From 943c12030ac66730f1a1ce2aa32e57a2b397064e Mon Sep 17 00:00:00 2001 From: aarzilli Date: Thu, 5 Nov 2015 18:25:52 +0100 Subject: [PATCH] proc/variables: map types support Use m[n:] to skip the first n keys of a map Map indexing is implemented with a linear scan Implements #61, #122 --- _fixtures/testvariables3.go | 48 +++++- proc/eval.go | 91 ++++++++++-- proc/variables.go | 258 ++++++++++++++++++++++++++++++++- service/api/prettyprint.go | 4 +- service/test/variables_test.go | 46 ++++++ 5 files changed, 425 insertions(+), 22 deletions(-) diff --git a/_fixtures/testvariables3.go b/_fixtures/testvariables3.go index a75a0be7..f4090684 100644 --- a/_fixtures/testvariables3.go +++ b/_fixtures/testvariables3.go @@ -45,11 +45,57 @@ func main() { var nilptr *int = nil ch1 := make(chan int, 2) var chnil chan int = nil + m1 := map[string]astruct{ + "Malone": astruct{2, 3}, + "Adenauer": astruct{}, + "squadrons": astruct{}, + "quintuplets": astruct{}, + "parasite": astruct{}, + "wristwatches": astruct{}, + "flashgun": astruct{}, + "equivocally": astruct{}, + "sweetbrier": astruct{}, + "idealism": astruct{}, + "tangos": astruct{}, + "alterable": astruct{}, + "quaffing": astruct{}, + "arsenic": astruct{}, + "coincidentally": astruct{}, + "hindrances": astruct{}, + "zoning": astruct{}, + "egging": astruct{}, + "inserts": astruct{}, + "adaptive": astruct{}, + "orientations": astruct{}, + "periling": astruct{}, + "lip": astruct{}, + "chant": astruct{}, + "availing": astruct{}, + "fern": astruct{}, + "flummoxes": astruct{}, + "meanders": astruct{}, + "ravenously": astruct{}, + "reminisce": astruct{}, + "snorkel": astruct{}, + "gutters": astruct{}, + "jibbed": astruct{}, + "tiara": astruct{}, + "takers": astruct{}, + "animates": astruct{}, + "Zubenelgenubi": astruct{}, + "bantering": astruct{}, + "tumblers": astruct{}, + "horticulturists": astruct{}, + "thallium": astruct{}, + } + var mnil map[string]astruct = nil + m2 := map[int]*astruct{1: &astruct{10, 11}} + m3 := map[astruct]int{{1, 1}: 42, {2, 2}: 43} var amb1 = 1 runtime.Breakpoint() for amb1 := 0; amb1 < 10; amb1++ { fmt.Println(amb1) } - fmt.Println(i1, i2, i3, p1, amb1, s1, a1, p2, p3, s2, as1, str1, f1, fn1, fn2, nilslice, nilptr, ch1, chnil) + fmt.Println(i1, i2, i3, p1, amb1, s1, a1, p2, p3, s2, as1, str1, f1, fn1, fn2, nilslice, nilptr, ch1, chnil, m1, mnil, m2, m3) } diff --git a/proc/eval.go b/proc/eval.go index c605d1f2..63ca079c 100644 --- a/proc/eval.go +++ b/proc/eval.go @@ -250,9 +250,6 @@ func (scope *EvalScope) evalIndex(node *ast.IndexExpr) (*Variable, error) { if xev.Unreadable != nil { return nil, xev.Unreadable } - if xev.base == 0 { - return nil, fmt.Errorf("can not index \"%s\"", exprToString(node.X)) - } idxev, err := scope.evalAST(node.Index) if err != nil { @@ -261,6 +258,9 @@ func (scope *EvalScope) evalIndex(node *ast.IndexExpr) (*Variable, error) { switch xev.Kind { case reflect.Slice, reflect.Array, reflect.String: + if xev.base == 0 { + return nil, fmt.Errorf("can not index \"%s\"", exprToString(node.X)) + } n, err := idxev.asInt() if err != nil { return nil, err @@ -268,7 +268,11 @@ func (scope *EvalScope) evalIndex(node *ast.IndexExpr) (*Variable, error) { return xev.sliceAccess(int(n)) case reflect.Map: - return nil, fmt.Errorf("map access not implemented") + idxev.loadValue() + if idxev.Unreadable != nil { + return nil, idxev.Unreadable + } + return xev.mapAccess(idxev) default: return nil, fmt.Errorf("invalid expression \"%s\" (type %s does not support indexing)", exprToString(node.X), xev.DwarfType.String()) @@ -285,16 +289,6 @@ func (scope *EvalScope) evalReslice(node *ast.SliceExpr) (*Variable, error) { if xev.Unreadable != nil { return nil, xev.Unreadable } - if xev.base == 0 { - return nil, fmt.Errorf("can not slice \"%s\"", exprToString(node.X)) - } - - switch xev.Kind { - case reflect.Slice, reflect.Array, reflect.String: - //ok - default: - return nil, fmt.Errorf("cannot slice \"%s\" (type %s)", exprToString(node.X), xev.DwarfType.String()) - } var low, high int64 @@ -322,8 +316,21 @@ func (scope *EvalScope) evalReslice(node *ast.SliceExpr) (*Variable, error) { } } - r, err := xev.reslice(low, high) - return r, err + switch xev.Kind { + case reflect.Slice, reflect.Array, reflect.String: + if xev.base == 0 { + return nil, fmt.Errorf("can not slice \"%s\"", exprToString(node.X)) + } + return xev.reslice(low, high) + case reflect.Map: + if node.High != nil { + return nil, fmt.Errorf("second slice argument must be empty for maps") + } + xev.mapSkip += int(low) + return xev, nil + default: + return nil, fmt.Errorf("can not slice \"%s\" (type %s)", exprToString(node.X), xev.DwarfType.String()) + } } // Evaluates a pointer dereference expression: * @@ -654,6 +661,24 @@ func (v *Variable) asInt() (int64, error) { return n, nil } +func (v *Variable) asUint() (uint64, error) { + if v.DwarfType == nil { + if v.Value.Kind() != constant.Int { + return 0, fmt.Errorf("can not convert constant %s to uint", v.Value) + } + } else { + v.loadValue() + if v.Unreadable != nil { + return 0, v.Unreadable + } + if _, ok := v.DwarfType.(*dwarf.UintType); !ok { + return 0, fmt.Errorf("can not convert value of type %s to uint", v.DwarfType.String()) + } + } + n, _ := constant.Uint64Val(v.Value) + return n, nil +} + func (v *Variable) isType(typ dwarf.Type, kind reflect.Kind) error { if v.DwarfType != nil { if typ != nil && typ.String() != v.RealType.String() { @@ -723,6 +748,40 @@ func (v *Variable) sliceAccess(idx int) (*Variable, error) { return newVariable("", v.base+uintptr(int64(idx)*v.stride), v.fieldType, v.thread), nil } +func (v *Variable) mapAccess(idx *Variable) (*Variable, error) { + it := v.mapIterator() + if it == nil { + return nil, fmt.Errorf("can not access unreadable map: %v", v.Unreadable) + } + + first := true + for it.next() { + key := it.key() + key.loadValue() + if key.Unreadable != nil { + return nil, fmt.Errorf("can not access unreadable map: %v", key.Unreadable) + } + if first { + first = false + if err := idx.isType(key.DwarfType, key.Kind); err != nil { + return nil, err + } + } + eql, err := compareOp(token.EQL, key, idx) + if err != nil { + return nil, err + } + if eql { + return it.value(), nil + } + } + if v.Unreadable != nil { + return nil, v.Unreadable + } + // go would return zero for the map value type here, we do not have the ability to create zeroes + return nil, fmt.Errorf("key not found") +} + func (v *Variable) reslice(low int64, high int64) (*Variable, error) { if low < 0 || low >= v.Len || high < 0 || high > v.Len { return nil, fmt.Errorf("index out of bounds") diff --git a/proc/variables.go b/proc/variables.go index 0710717e..428c8c70 100644 --- a/proc/variables.go +++ b/proc/variables.go @@ -23,6 +23,9 @@ const ( ChanRecv = "chan receive" ChanSend = "chan send" + + hashTophashEmpty = 0 // used by map reading code, indicates an empty bucket + hashMinTopHash = 4 // used by map reading code, indicates minimum value of tophash that isn't empty or evacuated ) // Represents a variable. @@ -42,12 +45,15 @@ type Variable struct { // base address of arrays, base address of the backing array for slices (0 for nil slices) // base address of the backing byte array for strings - // address of the struct backing a chan variable + // address of the struct backing chan and map variables // address of the function entry point for function variables (0 for nil function pointers) base uintptr stride int64 fieldType dwarf.Type + // number of elements to skip when loading a map + mapSkip int + Children []Variable loaded bool @@ -126,6 +132,8 @@ func newVariable(name string, addr uintptr, dwarfType dwarf.Type, thread *Thread structtyp, isstruct := t.Type.(*dwarf.StructType) if isstruct && strings.HasPrefix(structtyp.StructName, "hchan<") { v.Kind = reflect.Chan + } else if isstruct && strings.HasPrefix(structtyp.StructName, "hash<") { + v.Kind = reflect.Map } else { v.Kind = reflect.Ptr } @@ -716,6 +724,9 @@ func (v *Variable) loadValueInternal(recurseLevel int) { v.Len = sv.Len v.base = sv.Addr + case reflect.Map: + v.loadMap(recurseLevel) + case reflect.String: var val string val, v.Unreadable = v.thread.readStringValue(v.base, v.Len) @@ -762,8 +773,6 @@ func (v *Variable) loadValueInternal(recurseLevel int) { v.Value = constant.MakeFloat64(val) case reflect.Func: v.readFunctionPtr() - case reflect.Map: - fallthrough default: v.Unreadable = fmt.Errorf("unknown or unsupported kind: \"%s\"", v.Kind.String()) } @@ -1093,6 +1102,249 @@ func (v *Variable) readFunctionPtr() { v.Value = constant.MakeString(fn.Name) } +func (v *Variable) loadMap(recurseLevel int) { + it := v.mapIterator() + if it == nil { + return + } + + for skip := 0; skip < v.mapSkip; skip++ { + if ok := it.next(); !ok { + return + } + } + + count := 0 + errcount := 0 + for it.next() { + if count >= maxArrayValues { + break + } + key := it.key() + val := it.value() + key.loadValue() + val.loadValue() + if key.Unreadable != nil || val.Unreadable != nil { + errcount++ + } + v.Children = append(v.Children, *key) + v.Children = append(v.Children, *val) + count++ + if errcount > maxErrCount { + break + } + } +} + +type mapIterator struct { + v *Variable + numbuckets uint64 + oldmask uint64 + buckets *Variable + oldbuckets *Variable + b *Variable + bidx uint64 + + tophashes *Variable + keys *Variable + values *Variable + overflow *Variable + + idx int64 +} + +// Code derived from go/src/runtime/hashmap.go +func (v *Variable) mapIterator() *mapIterator { + sv := v.maybeDereference() + v.base = sv.Addr + + maptype, ok := sv.RealType.(*dwarf.StructType) + if !ok { + v.Unreadable = fmt.Errorf("wrong real type for map") + return nil + } + + it := &mapIterator{v: v, bidx: 0, b: nil, idx: 0} + + if sv.Addr == 0 { + it.numbuckets = 0 + return it + } + + for _, f := range maptype.Field { + var err error + field, _ := sv.toField(f) + switch f.Name { + case "count": + v.Len, err = field.asInt() + case "B": + var b uint64 + b, err = field.asUint() + it.numbuckets = 1 << b + it.oldmask = (1 << (b - 1)) - 1 + case "buckets": + it.buckets = field.maybeDereference() + case "oldbuckets": + it.oldbuckets = field.maybeDereference() + } + if err != nil { + v.Unreadable = err + return nil + } + } + + return it +} + +func (it *mapIterator) nextBucket() bool { + if it.overflow != nil && it.overflow.Addr > 0 { + it.b = it.overflow + } else { + it.b = nil + + for it.bidx < it.numbuckets { + it.b = it.buckets.clone() + it.b.Addr += uintptr(uint64(it.buckets.DwarfType.Size()) * it.bidx) + + if it.oldbuckets.Addr <= 0 { + break + } + + // if oldbuckets is not nil we are iterating through a map that is in + // the middle of a grow. + // if the bucket we are looking at hasn't been filled in we iterate + // instead through its corresponding "oldbucket" (i.e. the bucket the + // elements of this bucket are coming from) but only if this is the first + // of the two buckets being created from the same oldbucket (otherwise we + // would print some keys twice) + + oldbidx := it.bidx & it.oldmask + oldb := it.oldbuckets.clone() + oldb.Addr += uintptr(uint64(it.oldbuckets.DwarfType.Size()) * oldbidx) + + if mapEvacuated(oldb) { + break + } + + if oldbidx == it.bidx { + it.b = oldb + break + } + + // oldbucket origin for current bucket has not been evacuated but we have already + // iterated over it so we should just skip it + it.b = nil + it.bidx++ + } + + if it.b == nil { + return false + } + it.bidx++ + } + + if it.b.Addr <= 0 { + return false + } + + it.tophashes = nil + it.keys = nil + it.values = nil + it.overflow = nil + + for _, f := range it.b.DwarfType.(*dwarf.StructType).Field { + field, err := it.b.toField(f) + if err != nil { + it.v.Unreadable = err + return false + } + if field.Unreadable != nil { + it.v.Unreadable = field.Unreadable + return false + } + + switch f.Name { + case "tophash": + it.tophashes = field + case "keys": + it.keys = field + case "values": + it.values = field + case "overflow": + it.overflow = field.maybeDereference() + } + } + + // sanity checks + if it.tophashes == nil || it.keys == nil || it.values == nil { + it.v.Unreadable = fmt.Errorf("malformed map type") + return false + } + + if it.tophashes.Kind != reflect.Array || it.keys.Kind != reflect.Array || it.values.Kind != reflect.Array { + it.v.Unreadable = fmt.Errorf("malformed map type: keys, values or tophash of a bucket is not an array") + return false + } + + if it.tophashes.Len != it.keys.Len || it.tophashes.Len != it.values.Len { + it.v.Unreadable = fmt.Errorf("malformed map type: inconsistent array length in bucket") + return false + } + + return true +} + +func (it *mapIterator) next() bool { + for { + if it.b == nil || it.idx >= it.tophashes.Len { + r := it.nextBucket() + if !r { + return false + } + it.idx = 0 + } + tophash, _ := it.tophashes.sliceAccess(int(it.idx)) + h, err := tophash.asUint() + if err != nil { + it.v.Unreadable = fmt.Errorf("unreadable tophash: %v", err) + return false + } + it.idx++ + if h != hashTophashEmpty { + return true + } + } +} + +func (it *mapIterator) key() *Variable { + k, _ := it.keys.sliceAccess(int(it.idx - 1)) + return k +} + +func (it *mapIterator) value() *Variable { + v, _ := it.values.sliceAccess(int(it.idx - 1)) + return v +} + +func mapEvacuated(b *Variable) bool { + if b.Addr == 0 { + return true + } + for _, f := range b.DwarfType.(*dwarf.StructType).Field { + if f.Name != "tophash" { + continue + } + tophashes, _ := b.toField(f) + tophash0var, _ := tophashes.sliceAccess(0) + tophash0, err := tophash0var.asUint() + if err != nil { + return true + } + return tophash0 > hashTophashEmpty && tophash0 < hashMinTopHash + } + return true +} + // Fetches all variables of a specific type in the current function scope func (scope *EvalScope) variablesByTag(tag dwarf.Tag) ([]*Variable, error) { reader := scope.DwarfReader() diff --git a/service/api/prettyprint.go b/service/api/prettyprint.go index 66f042c3..2879d214 100644 --- a/service/api/prettyprint.go +++ b/service/api/prettyprint.go @@ -163,13 +163,13 @@ func (v *Variable) writeMapTo(buf *bytes.Buffer, newlines, includeType bool, ind } } - if len(v.Children) != int(v.Len) { + if len(v.Children)/2 != int(v.Len) { if nl { fmt.Fprintf(buf, "\n%s%s", indent, indentString) } else { fmt.Fprintf(buf, ",") } - fmt.Fprintf(buf, "...+%d more", int(v.Len)-len(v.Children)) + fmt.Fprintf(buf, "...+%d more", int(v.Len)-(len(v.Children)/2)) } if nl { diff --git a/service/test/variables_test.go b/service/test/variables_test.go index 1639563e..0ea9907f 100644 --- a/service/test/variables_test.go +++ b/service/test/variables_test.go @@ -389,6 +389,14 @@ func TestEvalExpression(t *testing.T) { {"ch1", true, "chan int 0/2", "", "chan int", nil}, {"ch1+1", false, "", "", "", fmt.Errorf("can not convert 1 constant to chan int")}, + // maps + {"m1[\"Malone\"]", false, "struct main.astruct {A: 2, B: 3}", "", "struct main.astruct", nil}, + {"m2[1].B", false, "11", "", "int", nil}, + {"m2[c1.sa[2].B-4].A", false, "10", "", "int", nil}, + {"m2[*p1].B", false, "11", "", "int", nil}, + {"m3[as1]", false, "42", "", "int", nil}, + {"mnil[\"Malone\"]", false, "", "", "", fmt.Errorf("key not found")}, + // combined expressions {"c1.pb.a.A", true, "1", "", "int", nil}, {"c1.sa[1].B", false, "3", "", "int", nil}, @@ -462,6 +470,9 @@ func TestEvalExpression(t *testing.T) { {"ch1 == nil", false, "false", "", "", nil}, {"chnil == nil", false, "true", "", "", nil}, {"ch1 == chnil", false, "", "", "", fmt.Errorf("can not compare chan variables")}, + {"m1 == nil", false, "false", "", "", nil}, + {"mnil == m1", false, "", "", "", fmt.Errorf("can not compare map variables")}, + {"mnil == nil", false, "true", "", "", nil}, // errors {"&3", false, "", "", "", fmt.Errorf("can not take address of \"3\"")}, @@ -521,3 +532,38 @@ func TestEvalAddrAndCast(t *testing.T) { assertVariable(t, a, varTest{aaddrstr, false, "struct main.astruct {A: 1, B: 2}", "", "struct main.astruct", nil}) }) } + +func TestMapEvaluation(t *testing.T) { + withTestProcess("testvariables3", t, func(p *proc.Process, fixture protest.Fixture) { + assertNoError(p.Continue(), t, "Continue() returned an error") + m1v, err := evalVariable(p, "m1") + assertNoError(err, t, "EvalVariable()") + m1 := api.ConvertVar(m1v) + t.Logf("m1 = %v", m1.MultilineString("")) + + if m1.Type != "map[string]main.astruct" { + t.Fatalf("Wrong type: %s", m1.Type) + } + + if len(m1.Children)/2 != 41 { + t.Fatalf("Wrong number of children: %d", len(m1.Children)/2) + } + + found := false + for i := range m1.Children { + if i%2 == 0 && m1.Children[i].Value == "Malone" { + found = true + } + } + if !found { + t.Fatalf("Could not find Malone") + } + + m1sliced, err := evalVariable(p, "m1[10:]") + assertNoError(err, t, "EvalVariable(m1[10:])") + if len(m1sliced.Children)/2 != int(m1.Len-10) { + t.Fatalf("Wrong number of children (after slicing): %d", len(m1sliced.Children)/2) + } + }) + +} -- GitLab