cmd/compile: de-virtualize interface calls

With this change, code like

    h := sha1.New()
    h.Write(buf)
    sum := h.Sum()

gets compiled into static calls rather than
interface calls, because the compiler is able
to prove that 'h' is really a *sha1.digest.

The InterCall re-write rule hits a few dozen times
during make.bash, and hundreds of times during all.bash.

The most common pattern identified by the compiler
is a constructor like

    func New() Interface { return &impl{...} }

where the constructor gets inlined into the caller,
and the result is used immediately. Examples include
{sha1,md5,crc32,crc64,...}.New, base64.NewEncoder,
base64.NewDecoder, errors.New, net.Pipe, and so on.

Some existing benchmarks that change on darwin/amd64:

Crc64/ISO4KB-8        2.67µs ± 1%    2.66µs ± 0%  -0.36%  (p=0.015 n=10+10)
Crc64/ISO1KB-8         694ns ± 0%     690ns ± 1%  -0.59%  (p=0.001 n=10+10)
Adler32KB-8            473ns ± 1%     471ns ± 0%  -0.39%  (p=0.010 n=10+9)

On architectures like amd64, the reduction in code size
appears to contribute more to benchmark improvements than just
removing the indirect call, since that branch gets predicted
accurately when called in a loop.

Updates golang#19361

Change-Id: Ia9d30afdd5f6b4d38d38b14b88f308acae8ce7ed

src/cmd/compile/internal/gc/reflect.go

@@ -16,6 +16,7 @@ import (
 type itabEntry struct {
 	t, itype *Type
 	sym      *Sym
+	concrete []*obj.LSym // itab.fun[x] is implemented by concrete[x]
 }
 
 type ptabEntry struct {
@@ -78,7 +79,7 @@ const (
 func structfieldSize() int       { return 3 * Widthptr } // Sizeof(runtime.structfield{})
 func imethodSize() int           { return 4 + 4 }        // Sizeof(runtime.imethod{})
 func uncommonSize(t *Type) int { // Sizeof(runtime.uncommontype{})
-	if t.Sym == nil && len(methods(t)) == 0 {
+	if t.Sym == nil && len(methods(t, true)) == 0 {
 		return 0
 	}
 	return 4 + 2 + 2 + 4 + 4
@@ -287,8 +288,8 @@ func methodfunc(f *Type, receiver *Type) *Type {
 }
 
 // methods returns the methods of the non-interface type t, sorted by name.
-// Generates stub functions as needed.
-func methods(t *Type) []*Sig {
+// If gen is true, generates stub functions as needed.
+func methods(t *Type, gen bool) []*Sig {
 	// method type
 	mt := methtype(t)
 
@@ -352,7 +353,7 @@ func methods(t *Type) []*Sig {
 		sig.type_ = methodfunc(f.Type, t)
 		sig.mtype = methodfunc(f.Type, nil)
 
-		if sig.isym.Flags&SymSiggen == 0 {
+		if gen && sig.isym.Flags&SymSiggen == 0 {
 			sig.isym.Flags |= SymSiggen
 			if !eqtype(this, it) || this.Width < Types[Tptr].Width {
 				compiling_wrappers = 1
@@ -361,7 +362,7 @@ func methods(t *Type) []*Sig {
 			}
 		}
 
-		if sig.tsym.Flags&SymSiggen == 0 {
+		if gen && sig.tsym.Flags&SymSiggen == 0 {
 			sig.tsym.Flags |= SymSiggen
 			if !eqtype(this, t) {
 				compiling_wrappers = 1
@@ -376,7 +377,8 @@ func methods(t *Type) []*Sig {
 }
 
 // imethods returns the methods of the interface type t, sorted by name.
-func imethods(t *Type) []*Sig {
+// If gen is false, then wrappers are not eagerly generated.
+func imethods(t *Type, gen bool) []*Sig {
 	var methods []*Sig
 	for _, f := range t.Fields().Slice() {
 		if f.Type.Etype != TFUNC || f.Sym == nil {
@@ -416,7 +418,7 @@ func imethods(t *Type) []*Sig {
 		// code can refer to it.
 		isym := methodsym(method, t, 0)
 
-		if isym.Flags&SymSiggen == 0 {
+		if gen && isym.Flags&SymSiggen == 0 {
 			isym.Flags |= SymSiggen
 			genwrapper(t, f, isym, 0)
 		}
@@ -602,7 +604,7 @@ func dname(name, tag string, pkg *Pkg, exported bool) *obj.LSym {
 // dataAdd is the offset in bytes after the header where the
 // backing array of the []method field is written (by dextratypeData).
 func dextratype(s *Sym, ot int, t *Type, dataAdd int) int {
-	m := methods(t)
+	m := methods(t, true)
 	if t.Sym == nil && len(m) == 0 {
 		return ot
 	}
@@ -653,7 +655,7 @@ func typePkg(t *Type) *Pkg {
 // runtime.uncommontype.
 func dextratypeData(s *Sym, ot int, t *Type) int {
 	lsym := Linksym(s)
-	for _, a := range methods(t) {
+	for _, a := range methods(t, true) {
 		// ../../../../runtime/type.go:/method
 		exported := exportname(a.name)
 		var pkg *Pkg
@@ -838,7 +840,7 @@ func dcommontype(s *Sym, ot int, t *Type) int {
 	var sptr *Sym
 	if !t.IsPtr() || t.ptrTo != nil {
 		tptr := ptrto(t)
-		if t.Sym != nil || methods(tptr) != nil {
+		if t.Sym != nil || methods(tptr, true) != nil {
 			sptrWeak = false
 		}
 		sptr = dtypesym(tptr)
@@ -992,7 +994,7 @@ func typename(t *Type) *Node {
 	return n
 }
 
-func itabname(t, itype *Type) *Node {
+func itabname(t, itype *Type, genimpl bool) *Node {
 	if t == nil || (t.IsPtr() && t.Elem() == nil) || t.IsUntyped() || !itype.IsInterface() || itype.IsEmptyInterface() {
 		Fatalf("itabname(%v, %v)", t, itype)
 	}
@@ -1004,7 +1006,12 @@ func itabname(t, itype *Type) *Node {
 		n.Typecheck = 1
 		s.Def = n
 
-		itabs = append(itabs, itabEntry{t: t, itype: itype, sym: s})
+		var concrete []*obj.LSym
+		if genimpl {
+			concrete = genfun(t, itype)
+		}
+
+		itabs = append(itabs, itabEntry{t: t, itype: itype, sym: s, concrete: concrete})
 	}
 
 	n := nod(OADDR, s.Def, nil)
@@ -1224,7 +1231,7 @@ ok:
 		}
 
 	case TINTER:
-		m := imethods(t)
+		m := imethods(t, true)
 		n := len(m)
 		for _, a := range m {
 			dtypesym(a.type_)
@@ -1379,6 +1386,65 @@ ok:
 	return s
 }
 
+// calculate the set of concrete implementations
+// for each entry in the itab
+// given by the type/interface pair
+func genfun(t, it *Type) []*obj.LSym {
+	if t == nil || it == nil {
+		return nil
+	}
+	sigs := imethods(it, false)
+	methods := methods(t, false)
+	out := make([]*obj.LSym, 0, len(sigs))
+	if len(sigs) == 0 {
+		return nil
+	}
+
+	// both sigs and methods are sorted by name,
+	// so we can find the intersect in a single pass
+	for _, m := range methods {
+		if m.name == sigs[0].name {
+			out = append(out, Linksym(m.isym))
+			sigs = sigs[1:]
+			if len(sigs) == 0 {
+				break
+			}
+		}
+	}
+
+	return out
+}
+
+// itabsym is used by the SSA backend
+// to convert interface calls into static calls;
+// it deliberately doesn't create
+// any additional Nodes, Syms, etc.
+func itabsym(t, it *Type, offset int64) *obj.LSym {
+	var syms []*obj.LSym
+
+	// TODO: something that isn't O(itabs)?
+	// We expect this lookup to occur
+	// infrequently in practice; perhaps
+	// this is offset by O(1) itab creation.
+	for i := range itabs {
+		e := &itabs[i]
+		if e.t == t && e.itype == it {
+			syms = e.concrete
+			break
+		}
+	}
+	if syms == nil {
+		return nil
+	}
+
+	// keep this arithmetic in sync with *itab layout
+	methodnum := int((offset - (3 * int64(Widthptr)) - 8) / int64(Widthptr))
+	if methodnum >= len(syms) {
+		return nil
+	}
+	return syms[methodnum]
+}
+
 func dumptypestructs() {
 	// copy types from externdcl list to signatlist
 	for _, n := range externdcl {
@@ -1417,10 +1483,10 @@ func dumptypestructs() {
 		// }
 		o := dsymptr(i.sym, 0, dtypesym(i.itype), 0)
 		o = dsymptr(i.sym, o, dtypesym(i.t), 0)
-		o += Widthptr                          // skip link field
-		o = duint32(i.sym, o, typehash(i.t))   // copy of type hash
-		o += 4                                 // skip bad/inhash/unused fields
-		o += len(imethods(i.itype)) * Widthptr // skip fun method pointers
+		o += Widthptr                                // skip link field
+		o = duint32(i.sym, o, typehash(i.t))         // copy of type hash
+		o += 4                                       // skip bad/inhash/unused fields
+		o += len(imethods(i.itype, true)) * Widthptr // skip fun method pointers
 		// at runtime the itab will contain pointers to types, other itabs and
 		// method functions. None are allocated on heap, so we can use obj.NOPTR.
 		ggloblsym(i.sym, int32(o), int16(obj.DUPOK|obj.NOPTR))

src/cmd/compile/internal/gc/sinit.go

@@ -502,7 +502,7 @@ func staticassign(l *Node, r *Node, out *[]*Node) bool {
 		if l.Type.IsEmptyInterface() {
 			itab = typename(val.Type)
 		} else {
-			itab = itabname(val.Type, l.Type)
+			itab = itabname(val.Type, l.Type, true)
 		}
 
 		// Create a copy of l to modify while we emit data.

src/cmd/compile/internal/gc/ssa.go

@@ -392,6 +392,10 @@ func (s *state) newValue2I(op ssa.Op, t ssa.Type, aux int64, arg0, arg1 *ssa.Val
 	return s.curBlock.NewValue2I(s.peekPos(), op, t, aux, arg0, arg1)
 }
 
+func (s *state) newValue2A(op ssa.Op, t ssa.Type, aux interface{}, arg0, arg1 *ssa.Value) *ssa.Value {
+	return s.curBlock.NewValue2A(s.peekPos(), op, t, aux, arg0, arg1)
+}
+
 // newValue3 adds a new value with three arguments to the current block.
 func (s *state) newValue3(op ssa.Op, t ssa.Type, arg0, arg1, arg2 *ssa.Value) *ssa.Value {
 	return s.curBlock.NewValue3(s.peekPos(), op, t, arg0, arg1, arg2)
@@ -2038,7 +2042,7 @@ func (s *state) expr(n *Node) *ssa.Value {
 	case OEFACE:
 		tab := s.expr(n.Left)
 		data := s.expr(n.Right)
-		return s.newValue2(ssa.OpIMake, n.Type, tab, data)
+		return s.newValue2A(ssa.OpIMake, n.Type, n.Right.Type, tab, data)
 
 	case OSLICE, OSLICEARR, OSLICE3, OSLICE3ARR:
 		v := s.expr(n.Left)
@@ -4079,7 +4083,10 @@ func (s *state) dottype(n *Node, commaok bool) (res, resok *ssa.Value) {
 		targetITab = target
 	} else {
 		// Looking for pointer to itab for target type and source interface.
-		targetITab = s.expr(itabname(n.Type, n.Left.Type))
+		// Don't generate the implementation functions; there's no
+		// de-virtualization opportunity here, and we're already
+		// compiling a function.
+		targetITab = s.expr(itabname(n.Type, n.Left.Type, false))
 	}
 
 	var tmp *Node       // temporary for use with large types
@@ -4873,6 +4880,21 @@ func (e *ssaExport) SplitArray(name ssa.LocalSlot) ssa.LocalSlot {
 	return ssa.LocalSlot{N: n, Type: et, Off: name.Off}
 }
 
+func (e *ssaExport) DerefItab(st, sitab ssa.Type, offset int64) interface{} {
+	t, ok := st.(*Type)
+	if !ok {
+		return nil
+	}
+	itab, ok := sitab.(*Type)
+	if !ok {
+		return nil
+	}
+	if lsym := itabsym(t, itab, offset); lsym != nil {
+		return lsym
+	}
+	return nil
+}
+
 // namedAuto returns a new AUTO variable with the given name and type.
 // These are exposed to the debugger.
 func (e *ssaExport) namedAuto(name string, typ ssa.Type) ssa.GCNode {

src/cmd/compile/internal/gc/walk.go

@@ -386,6 +386,10 @@ func isSmallMakeSlice(n *Node) bool {
 func walkexprlist(s []*Node, init *Nodes) {
 	for i := range s {
 		s[i] = walkexpr(s[i], init)
+		if Curfn == nil {
+			println(s[i].String())
+			panic("curfn is nil")
+		}
 	}
 }
 
@@ -884,7 +888,7 @@ opswitch:
 			if n.Type.IsEmptyInterface() {
 				t = typename(n.Left.Type)
 			} else {
-				t = itabname(n.Left.Type, n.Type)
+				t = itabname(n.Left.Type, n.Type, true)
 			}
 			l := nod(OEFACE, t, n.Left)
 			l.Type = n.Type
@@ -932,7 +936,7 @@ opswitch:
 			if n.Type.IsEmptyInterface() {
 				t = typename(n.Left.Type)
 			} else {
-				t = itabname(n.Left.Type, n.Type)
+				t = itabname(n.Left.Type, n.Type, true)
 			}
 			l := nod(OEFACE, t, typecheck(nod(OADDR, value, nil), Erv))
 			l.Type = n.Type
@@ -978,7 +982,7 @@ opswitch:
 			if n.Left.Type.IsInterface() {
 				ll = append(ll, typename(n.Type))
 			} else {
-				ll = append(ll, itabname(n.Left.Type, n.Type))
+				ll = append(ll, itabname(n.Left.Type, n.Type, false))
 			}
 		}
 

src/cmd/compile/internal/ssa/check.go

@@ -158,6 +158,13 @@ func checkFunc(f *Func) {
 				}
 				canHaveAuxInt = true
 				canHaveAux = true
+			case auxTyp:
+				canHaveAux = true
+				if v.Aux != nil {
+					if _, ok := v.Aux.(Type); !ok {
+						f.Fatalf("auxTyp %T not a Type", v.Aux)
+					}
+				}
 			default:
 				f.Fatalf("unknown aux type for %s", v.Op)
 			}

src/cmd/compile/internal/ssa/config.go

@@ -121,6 +121,12 @@ type Frontend interface {
 	SplitArray(LocalSlot) LocalSlot              // array must be length 1
 	SplitInt64(LocalSlot) (LocalSlot, LocalSlot) // returns (hi, lo)
 
+	// DerefItab dereferences an itab
+	// entry, given the concrete type and
+	// the interface type and the offset of
+	// the method
+	DerefItab(t, it Type, offset int64) interface{} // returns *obj.LSym
+
 	// Line returns a string describing the given position.
 	Line(src.XPos) string
 

src/cmd/compile/internal/ssa/export_test.go

@@ -84,21 +84,22 @@ func (d DummyFrontend) Warnl(_ src.XPos, msg string, args ...interface{})  { d.t
 func (d DummyFrontend) Debug_checknil() bool                               { return false }
 func (d DummyFrontend) Debug_wb() bool                                     { return false }
 
-func (d DummyFrontend) TypeBool() Type    { return TypeBool }
-func (d DummyFrontend) TypeInt8() Type    { return TypeInt8 }
-func (d DummyFrontend) TypeInt16() Type   { return TypeInt16 }
-func (d DummyFrontend) TypeInt32() Type   { return TypeInt32 }
-func (d DummyFrontend) TypeInt64() Type   { return TypeInt64 }
-func (d DummyFrontend) TypeUInt8() Type   { return TypeUInt8 }
-func (d DummyFrontend) TypeUInt16() Type  { return TypeUInt16 }
-func (d DummyFrontend) TypeUInt32() Type  { return TypeUInt32 }
-func (d DummyFrontend) TypeUInt64() Type  { return TypeUInt64 }
-func (d DummyFrontend) TypeFloat32() Type { return TypeFloat32 }
-func (d DummyFrontend) TypeFloat64() Type { return TypeFloat64 }
-func (d DummyFrontend) TypeInt() Type     { return TypeInt64 }
-func (d DummyFrontend) TypeUintptr() Type { return TypeUInt64 }
-func (d DummyFrontend) TypeString() Type  { panic("unimplemented") }
-func (d DummyFrontend) TypeBytePtr() Type { return TypeBytePtr }
+func (d DummyFrontend) TypeBool() Type                             { return TypeBool }
+func (d DummyFrontend) TypeInt8() Type                             { return TypeInt8 }
+func (d DummyFrontend) TypeInt16() Type                            { return TypeInt16 }
+func (d DummyFrontend) TypeInt32() Type                            { return TypeInt32 }
+func (d DummyFrontend) TypeInt64() Type                            { return TypeInt64 }
+func (d DummyFrontend) TypeUInt8() Type                            { return TypeUInt8 }
+func (d DummyFrontend) TypeUInt16() Type                           { return TypeUInt16 }
+func (d DummyFrontend) TypeUInt32() Type                           { return TypeUInt32 }
+func (d DummyFrontend) TypeUInt64() Type                           { return TypeUInt64 }
+func (d DummyFrontend) TypeFloat32() Type                          { return TypeFloat32 }
+func (d DummyFrontend) TypeFloat64() Type                          { return TypeFloat64 }
+func (d DummyFrontend) TypeInt() Type                              { return TypeInt64 }
+func (d DummyFrontend) TypeUintptr() Type                          { return TypeUInt64 }
+func (d DummyFrontend) TypeString() Type                           { panic("unimplemented") }
+func (d DummyFrontend) TypeBytePtr() Type                          { return TypeBytePtr }
+func (d DummyFrontend) DerefItab(a, b Type, off int64) interface{} { return nil }
 
 func (d DummyFrontend) CanSSA(t Type) bool {
 	// There are no un-SSAable types in dummy land.

src/cmd/compile/internal/ssa/func.go

@@ -294,6 +294,19 @@ func (b *Block) NewValue2I(pos src.XPos, op Op, t Type, auxint int64, arg0, arg1
 	return v
 }
 
+// NewValue2A returns a new value in the block with two arguments and an aux value.
+func (b *Block) NewValue2A(pos src.XPos, op Op, t Type, aux interface{}, arg0, arg1 *Value) *Value {
+	v := b.Func.newValue(op, t, b, pos)
+	v.AuxInt = 0
+	v.Aux = aux
+	v.Args = v.argstorage[:2]
+	v.argstorage[0] = arg0
+	v.argstorage[1] = arg1
+	arg0.Uses++
+	arg1.Uses++
+	return v
+}
+
 // NewValue3 returns a new value in the block with three arguments and zero aux values.
 func (b *Block) NewValue3(pos src.XPos, op Op, t Type, arg0, arg1, arg2 *Value) *Value {
 	v := b.Func.newValue(op, t, b, pos)

src/cmd/compile/internal/ssa/gen/generic.rules

@@ -1424,3 +1424,7 @@
 	&& c == config.ctxt.FixedFrameSize() + config.RegSize // offset of return value
 	&& warnRule(config.Debug_checknil() && v.Pos.Line() > 1, v, "removed nil check")
 	-> (Invalid)
+
+// De-virtualize interface calls
+(InterCall [argsize] (Load (OffPtr [off] (ITab (IMake <it> {ctype} (Addr (SB)) z))) _) mem) && devirt(v, ctype, it, off) != nil ->
+	(StaticCall [argsize] {devirt(v, ctype, it, off)} cleanIntercall(mem, z))