proc: read G struct offset from runtime.tlsg if possible (#883)

When a Go program is externally linked, the external linker is
responsible for picking the TLS offset. It records its decision in the
runtime.tlsg symbol. Read the offset from that rather than guessing -16.

This implementation causes a regression: 1.4 and earlier will no longer
work.

pkg/proc/arch.go

@@ -3,11 +3,9 @@ package proc
 // Arch defines an interface for representing a
 // CPU architecture.
 type Arch interface {
-	SetGStructOffset(ver GoVersion, iscgo bool)
 	PtrSize() int
 	BreakpointInstruction() []byte
 	BreakpointSize() int
-	GStructOffset() uint64
 	DerefTLS() bool
 }
 
@@ -35,26 +33,6 @@ func AMD64Arch(goos string) *AMD64 {
 	}
 }
 
-// SetGStructOffset sets the offset of the G struct on the AMD64
-// arch struct. The offset is dependent on the Go compiler Version
-// and whether or not the target program was externally linked.
-func (a *AMD64) SetGStructOffset(ver GoVersion, isextld bool) {
-	switch a.goos {
-	case "darwin":
-		a.gStructOffset = 0x8a0
-	case "linux":
-		a.gStructOffset = 0xfffffffffffffff0
-		if isextld || ver.AfterOrEqual(GoVersion{1, 5, -1, 2, 0, ""}) || ver.IsDevel() {
-			a.gStructOffset += 8
-		}
-	case "windows":
-		// Use ArbitraryUserPointer (0x28) as pointer to pointer
-		// to G struct per:
-		// https://golang.org/src/runtime/cgo/gcc_windows_amd64.c
-		a.gStructOffset = 0x28
-	}
-}
-
 // PtrSize returns the size of a pointer
 // on this architecture.
 func (a *AMD64) PtrSize() int {
@@ -73,12 +51,6 @@ func (a *AMD64) BreakpointSize() int {
 	return a.breakInstructionLen
 }
 
-// GStructOffset returns the offset of the G
-// struct in thread local storage.
-func (a *AMD64) GStructOffset() uint64 {
-	return a.gStructOffset
-}
-
 // If DerefTLS returns true the value of regs.TLS()+GStructOffset() is a
 // pointer to the G struct
 func (a *AMD64) DerefTLS() bool {

pkg/proc/bininfo.go

@@ -28,13 +28,14 @@ type BinaryInfo struct {
 	// Maps package names to package paths, needed to lookup types inside DWARF info
 	packageMap map[string]string
 
-	Arch         Arch
-	dwarf        *dwarf.Data
-	frameEntries frame.FrameDescriptionEntries
-	lineInfo     line.DebugLines
-	goSymTable   *gosym.Table
-	types        map[string]dwarf.Offset
-	functions    []functionDebugInfo
+	Arch          Arch
+	dwarf         *dwarf.Data
+	frameEntries  frame.FrameDescriptionEntries
+	lineInfo      line.DebugLines
+	goSymTable    *gosym.Table
+	types         map[string]dwarf.Offset
+	functions     []functionDebugInfo
+	gStructOffset uint64
 
 	loadModuleDataOnce sync.Once
 	moduleData         []moduleData
@@ -74,6 +75,12 @@ func (bininfo *BinaryInfo) LoadBinaryInfo(path string, wg *sync.WaitGroup) error
 	return errors.New("unsupported operating system")
 }
 
+// GStructOffset returns the offset of the G
+// struct in thread local storage.
+func (bi *BinaryInfo) GStructOffset() uint64 {
+	return bi.gStructOffset
+}
+
 func (bi *BinaryInfo) LastModified() time.Time {
 	return bi.lastModified
 }
@@ -141,11 +148,12 @@ func (bi *BinaryInfo) LoadBinaryInfoElf(path string, wg *sync.WaitGroup) error {
 		return err
 	}
 
-	wg.Add(4)
+	wg.Add(5)
 	go bi.parseDebugFrameElf(elfFile, wg)
 	go bi.obtainGoSymbolsElf(elfFile, wg)
 	go bi.parseDebugLineInfoElf(elfFile, wg)
 	go bi.loadDebugInfoMaps(wg)
+	go bi.setGStructOffsetElf(elfFile, wg)
 	return nil
 }
 
@@ -224,6 +232,44 @@ func (bi *BinaryInfo) parseDebugLineInfoElf(exe *elf.File, wg *sync.WaitGroup) {
 	}
 }
 
+func (bi *BinaryInfo) setGStructOffsetElf(exe *elf.File, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	// This is a bit arcane. Essentially:
+	// - If the program is pure Go, it can do whatever it wants, and puts the G
+	//   pointer at %fs-8.
+	// - Otherwise, Go asks the external linker to place the G pointer by
+	//   emitting runtime.tlsg, a TLS symbol, which is relocated to the chosen
+	//   offset in libc's TLS block.
+	symbols, err := exe.Symbols()
+	if err != nil {
+		fmt.Println("could not parse ELF symbols", err)
+		os.Exit(1)
+	}
+	var tlsg *elf.Symbol
+	for _, symbol := range symbols {
+		if symbol.Name == "runtime.tlsg" {
+			s := symbol
+			tlsg = &s
+			break
+		}
+	}
+	if tlsg == nil {
+		bi.gStructOffset = ^uint64(8) + 1 // -8
+		return
+	}
+	var tls *elf.Prog
+	for _, prog := range exe.Progs {
+		if prog.Type == elf.PT_TLS {
+			tls = prog
+			break
+		}
+	}
+	// The TLS register points to the end of the TLS block, which is
+	// tls.Memsz long. runtime.tlsg is an offset from the beginning of that block.
+	bi.gStructOffset = ^(tls.Memsz) + 1 + tlsg.Value // -tls.Memsz + tlsg.Value
+}
+
 // PE ////////////////////////////////////////////////////////////////
 
 func (bi *BinaryInfo) LoadBinaryInfoPE(path string, wg *sync.WaitGroup) error {
@@ -245,6 +291,12 @@ func (bi *BinaryInfo) LoadBinaryInfoPE(path string, wg *sync.WaitGroup) error {
 	go bi.obtainGoSymbolsPE(peFile, wg)
 	go bi.parseDebugLineInfoPE(peFile, wg)
 	go bi.loadDebugInfoMaps(wg)
+
+	// Use ArbitraryUserPointer (0x28) as pointer to pointer
+	// to G struct per:
+	// https://golang.org/src/runtime/cgo/gcc_windows_amd64.c
+
+	bi.gStructOffset = 0x28
 	return nil
 }
 
@@ -444,6 +496,7 @@ func (bi *BinaryInfo) LoadBinaryInfoMacho(path string, wg *sync.WaitGroup) error
 	go bi.obtainGoSymbolsMacho(exe, wg)
 	go bi.parseDebugLineInfoMacho(exe, wg)
 	go bi.loadDebugInfoMaps(wg)
+	bi.gStructOffset = 0x8a0
 	return nil
 }
 

pkg/proc/core/core.go

@@ -178,13 +178,6 @@ func OpenCore(corePath, exePath string) (*Process, error) {
 		p.currentThread = th
 		break
 	}
-
-	ver, isextld, err := proc.GetGoInformation(p)
-	if err != nil {
-		return nil, err
-	}
-
-	p.bi.Arch.SetGStructOffset(ver, isextld)
 	p.selectedGoroutine, _ = proc.GetG(p.CurrentThread())
 
 	return p, nil

pkg/proc/gdbserial/gdbserver.go

@@ -62,6 +62,7 @@
 package gdbserial
 
 import (
+	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -258,6 +259,14 @@ func (p *Process) Connect(conn net.Conn, path string, pid int) error {
 		}
 	}
 
+	var wg sync.WaitGroup
+	err = p.bi.LoadBinaryInfo(path, &wg)
+	if err != nil {
+		conn.Close()
+		return err
+	}
+	wg.Wait()
+
 	// None of the stubs we support returns the value of fs_base or gs_base
 	// along with the registers, therefore we have to resort to executing a MOV
 	// instruction on the inferior to find out where the G struct of a given
@@ -272,14 +281,6 @@ func (p *Process) Connect(conn net.Conn, path string, pid int) error {
 		}
 	}
 
-	var wg sync.WaitGroup
-	err = p.bi.LoadBinaryInfo(path, &wg)
-	if err != nil {
-		conn.Close()
-		return err
-	}
-	wg.Wait()
-
 	err = p.updateThreadList(&threadUpdater{p: p})
 	if err != nil {
 		conn.Close()
@@ -296,14 +297,6 @@ func (p *Process) Connect(conn net.Conn, path string, pid int) error {
 		}
 	}
 
-	ver, isextld, err := proc.GetGoInformation(p)
-	if err != nil {
-		conn.Close()
-		p.bi.Close()
-		return err
-	}
-
-	p.bi.Arch.SetGStructOffset(ver, isextld)
 	p.selectedGoroutine, _ = proc.GetG(p.CurrentThread())
 
 	panicpc, err := proc.FindFunctionLocation(p, "runtime.startpanic", true, 0)
@@ -1158,27 +1151,24 @@ func (t *Thread) Blocked() bool {
 // OS/architecture that can be executed to load the address of G from an
 // inferior's thread.
 func (p *Process) loadGInstr() []byte {
+	var op []byte
 	switch p.bi.GOOS {
 	case "windows":
-		// mov rcx, QWORD PTR gs:0x28
-		return []byte{0x65, 0x48, 0x8b, 0x0c, 0x25, 0x28, 0x00, 0x00, 0x00}
+		// mov rcx, QWORD PTR gs:{uint32(off)}
+		op = []byte{0x65, 0x48, 0x8b, 0x0c, 0x25}
 	case "linux":
-		switch p.bi.Arch.GStructOffset() {
-		case 0xfffffffffffffff8, 0x0:
-			// mov    rcx,QWORD PTR fs:0xfffffffffffffff8
-			return []byte{0x64, 0x48, 0x8B, 0x0C, 0x25, 0xF8, 0xFF, 0xFF, 0xFF}
-		case 0xfffffffffffffff0:
-			// mov    rcx,QWORD PTR fs:0xfffffffffffffff0
-			return []byte{0x64, 0x48, 0x8B, 0x0C, 0x25, 0xF0, 0xFF, 0xFF, 0xFF}
-		default:
-			panic("not implemented")
-		}
+		// mov rcx,QWORD PTR fs:{uint32(off)}
+		op = []byte{0x64, 0x48, 0x8B, 0x0C, 0x25}
 	case "darwin":
-		// mov    rcx,QWORD PTR gs:0x8a0
-		return []byte{0x65, 0x48, 0x8B, 0x0C, 0x25, 0xA0, 0x08, 0x00, 0x00}
+		// mov rcx,QWORD PTR gs:{uint32(off)}
+		op = []byte{0x65, 0x48, 0x8B, 0x0C, 0x25}
 	default:
 		panic("unsupported operating system attempting to find Goroutine on Thread")
 	}
+	buf := &bytes.Buffer{}
+	buf.Write(op)
+	binary.Write(buf, binary.LittleEndian, uint32(p.bi.GStructOffset()))
+	return buf.Bytes()
 }
 
 // reloadRegisters loads the current value of the thread's registers.

pkg/proc/native/proc.go

@@ -398,12 +398,6 @@ func initializeDebugProcess(dbp *Process, path string, attach bool) (*Process, e
 		return nil, err
 	}
 
-	ver, isextld, err := proc.GetGoInformation(dbp)
-	if err != nil {
-		return nil, err
-	}
-
-	dbp.bi.Arch.SetGStructOffset(ver, isextld)
 	// selectedGoroutine can not be set correctly by the call to updateThreadList
 	// because without calling SetGStructOffset we can not read the G struct of currentThread
 	// but without calling updateThreadList we can not examine memory to determine

pkg/proc/proc.go

@@ -9,7 +9,6 @@ import (
 	"go/token"
 	"path/filepath"
 	"strconv"
-	"strings"
 
 	"golang.org/x/debug/dwarf"
 )
@@ -430,36 +429,22 @@ func GoroutinesInfo(dbp Process) ([]*G, error) {
 	return allg, nil
 }
 
-func GetGoInformation(p Process) (ver GoVersion, isextld bool, err error) {
+func GetGoVersion(p Process) (GoVersion, error) {
 	scope := &EvalScope{0, 0, p.CurrentThread(), nil, p.BinInfo(), 0}
 	vv, err := scope.packageVarAddr("runtime.buildVersion")
 	if err != nil {
-		return ver, false, fmt.Errorf("Could not determine version number: %v", err)
+		return GoVersion{}, fmt.Errorf("could not determine version number: %v", err)
 	}
 	vv.loadValue(LoadConfig{true, 0, 64, 0, 0})
 	if vv.Unreadable != nil {
-		err = fmt.Errorf("Unreadable version number: %v\n", vv.Unreadable)
-		return
+		return GoVersion{}, fmt.Errorf("unreadable version number: %v\n", vv.Unreadable)
 	}
 
 	ver, ok := ParseVersionString(constant.StringVal(vv.Value))
 	if !ok {
-		err = fmt.Errorf("Could not parse version number: %v\n", vv.Value)
-		return
+		return GoVersion{}, fmt.Errorf("could not parse version number: %v\n", vv.Value)
 	}
-
-	rdr := scope.BinInfo.DwarfReader()
-	rdr.Seek(0)
-	for entry, err := rdr.NextCompileUnit(); entry != nil; entry, err = rdr.NextCompileUnit() {
-		if err != nil {
-			return ver, isextld, err
-		}
-		if prod, ok := entry.Val(dwarf.AttrProducer).(string); ok && (strings.HasPrefix(prod, "GNU AS")) {
-			isextld = true
-			break
-		}
-	}
-	return
+	return ver, nil
 }
 
 // FindGoroutine returns a G struct representing the goroutine

pkg/proc/threads.go

@@ -302,29 +302,22 @@ func setStepIntoBreakpoint(dbp Process, text []AsmInstruction, cond ast.Expr) er
 }
 
 func getGVariable(thread Thread) (*Variable, error) {
-	arch := thread.Arch()
 	regs, err := thread.Registers(false)
 	if err != nil {
 		return nil, err
 	}
 
-	if arch.GStructOffset() == 0 {
-		// GetG was called through SwitchThread / updateThreadList during initialization
-		// thread.dbp.arch isn't setup yet (it needs a current thread to read global variables from)
-		return nil, fmt.Errorf("g struct offset not initialized")
-	}
-
 	gaddr, hasgaddr := regs.GAddr()
 	if !hasgaddr {
-		gaddrbs := make([]byte, arch.PtrSize())
-		_, err := thread.ReadMemory(gaddrbs, uintptr(regs.TLS()+arch.GStructOffset()))
+		gaddrbs := make([]byte, thread.Arch().PtrSize())
+		_, err := thread.ReadMemory(gaddrbs, uintptr(regs.TLS()+thread.BinInfo().GStructOffset()))
 		if err != nil {
 			return nil, err
 		}
 		gaddr = binary.LittleEndian.Uint64(gaddrbs)
 	}
 
-	return newGVariable(thread, uintptr(gaddr), arch.DerefTLS())
+	return newGVariable(thread, uintptr(gaddr), thread.Arch().DerefTLS())
 }
 
 func newGVariable(thread Thread, gaddr uintptr, deref bool) (*Variable, error) {