cmd/compile: de-virtualize interface calls

With this change, code like

    h := sha1.New()
    h.Write(buf)
    sum := h.Sum()

gets compiled into static calls rather than
interface calls, because the compiler is able
to prove that 'h' is really a *sha1.digest.

The InterCall re-write rule hits a few dozen times
during make.bash, and hundreds of times during all.bash.

The most common pattern identified by the compiler
is a constructor like

    func New() Interface { return &impl{...} }

where the constructor gets inlined into the caller,
and the result is used immediately. Examples include
{sha1,md5,crc32,crc64,...}.New, base64.NewEncoder,
base64.NewDecoder, errors.New, net.Pipe, and so on.

Some existing benchmarks that change on darwin/amd64:

Crc64/ISO4KB-8        2.67µs ± 1%    2.66µs ± 0%  -0.36%  (p=0.015 n=10+10)
Crc64/ISO1KB-8         694ns ± 0%     690ns ± 1%  -0.59%  (p=0.001 n=10+10)
Adler32KB-8            473ns ± 1%     471ns ± 0%  -0.39%  (p=0.010 n=10+9)

On architectures like amd64, the reduction in code size
appears to contribute more to benchmark improvements than just
removing the indirect call, since that branch gets predicted
accurately when called in a loop.

Updates golang#19361

Change-Id: Ia9d30afdd5f6b4d38d38b14b88f308acae8ce7ed

src/cmd/compile/internal/gc/reflect.go

@@ -16,6 +16,11 @@ import (
 type itabEntry struct {
 	t, itype *Type
 	sym      *Sym
+	lsym     *obj.LSym
+
+	// symbols of each method in
+	// the itab, sorted by byte offset
+	entries []*obj.LSym
 }
 
 type ptabEntry struct {
@@ -78,7 +83,7 @@ const (
 func structfieldSize() int       { return 3 * Widthptr } // Sizeof(runtime.structfield{})
 func imethodSize() int           { return 4 + 4 }        // Sizeof(runtime.imethod{})
 func uncommonSize(t *Type) int { // Sizeof(runtime.uncommontype{})
-	if t.Sym == nil && len(methods(t)) == 0 {
+	if t.Sym == nil && len(methods(t, true)) == 0 {
 		return 0
 	}
 	return 4 + 2 + 2 + 4 + 4
@@ -287,8 +292,8 @@ func methodfunc(f *Type, receiver *Type) *Type {
 }
 
 // methods returns the methods of the non-interface type t, sorted by name.
-// Generates stub functions as needed.
-func methods(t *Type) []*Sig {
+// If gen is true, generates stub functions as needed.
+func methods(t *Type, gen bool) []*Sig {
 	// method type
 	mt := methtype(t)
 
@@ -352,7 +357,7 @@ func methods(t *Type) []*Sig {
 		sig.type_ = methodfunc(f.Type, t)
 		sig.mtype = methodfunc(f.Type, nil)
 
-		if !sig.isym.Siggen() {
+		if gen && !sig.isym.Siggen() {
 			sig.isym.SetSiggen(true)
 			if !eqtype(this, it) || this.Width < Types[Tptr].Width {
 				compiling_wrappers = 1
@@ -361,7 +366,7 @@ func methods(t *Type) []*Sig {
 			}
 		}
 
-		if !sig.tsym.Siggen() {
+		if gen && !sig.tsym.Siggen() {
 			sig.tsym.SetSiggen(true)
 			if !eqtype(this, t) {
 				compiling_wrappers = 1
@@ -376,7 +381,8 @@ func methods(t *Type) []*Sig {
 }
 
 // imethods returns the methods of the interface type t, sorted by name.
-func imethods(t *Type) []*Sig {
+// If gen is false, then wrappers are not eagerly generated.
+func imethods(t *Type, gen bool) []*Sig {
 	var methods []*Sig
 	for _, f := range t.Fields().Slice() {
 		if f.Type.Etype != TFUNC || f.Sym == nil {
@@ -416,7 +422,7 @@ func imethods(t *Type) []*Sig {
 		// code can refer to it.
 		isym := methodsym(method, t, 0)
 
-		if !isym.Siggen() {
+		if gen && !isym.Siggen() {
 			isym.SetSiggen(true)
 			genwrapper(t, f, isym, 0)
 		}
@@ -602,7 +608,7 @@ func dname(name, tag string, pkg *Pkg, exported bool) *obj.LSym {
 // dataAdd is the offset in bytes after the header where the
 // backing array of the []method field is written (by dextratypeData).
 func dextratype(s *Sym, ot int, t *Type, dataAdd int) int {
-	m := methods(t)
+	m := methods(t, true)
 	if t.Sym == nil && len(m) == 0 {
 		return ot
 	}
@@ -653,7 +659,7 @@ func typePkg(t *Type) *Pkg {
 // runtime.uncommontype.
 func dextratypeData(s *Sym, ot int, t *Type) int {
 	lsym := Linksym(s)
-	for _, a := range methods(t) {
+	for _, a := range methods(t, true) {
 		// ../../../../runtime/type.go:/method
 		exported := exportname(a.name)
 		var pkg *Pkg
@@ -838,7 +844,7 @@ func dcommontype(s *Sym, ot int, t *Type) int {
 	var sptr *Sym
 	if !t.IsPtr() || t.ptrTo != nil {
 		tptr := ptrto(t)
-		if t.Sym != nil || methods(tptr) != nil {
+		if t.Sym != nil || methods(tptr, true) != nil {
 			sptrWeak = false
 		}
 		sptr = dtypesym(tptr)
@@ -992,7 +998,7 @@ func typename(t *Type) *Node {
 	return n
 }
 
-func itabname(t, itype *Type) *Node {
+func itabname(t, itype *Type, genimpl bool) *Node {
 	if t == nil || (t.IsPtr() && t.Elem() == nil) || t.IsUntyped() || !itype.IsInterface() || itype.IsEmptyInterface() {
 		Fatalf("itabname(%v, %v)", t, itype)
 	}
@@ -1004,7 +1010,18 @@ func itabname(t, itype *Type) *Node {
 		n.Typecheck = 1
 		s.Def = n
 
-		itabs = append(itabs, itabEntry{t: t, itype: itype, sym: s})
+		var methods []*obj.LSym
+		if genimpl {
+			methods = genfun(t, itype)
+		}
+
+		itabs = append(itabs, itabEntry{
+			t:       t,
+			itype:   itype,
+			sym:     s,
+			lsym:    Linksym(s),
+			entries: methods,
+		})
 	}
 
 	n := nod(OADDR, s.Def, nil)
@@ -1224,7 +1241,7 @@ ok:
 		}
 
 	case TINTER:
-		m := imethods(t)
+		m := imethods(t, true)
 		n := len(m)
 		for _, a := range m {
 			dtypesym(a.type_)
@@ -1379,6 +1396,64 @@ ok:
 	return s
 }
 
+// calculate the set of concrete implementations
+// for each entry in the itab
+// given by the type/interface pair
+func genfun(t, it *Type) []*obj.LSym {
+	if t == nil || it == nil {
+		return nil
+	}
+	sigs := imethods(it, false)
+	methods := methods(t, false)
+	out := make([]*obj.LSym, 0, len(sigs))
+	if len(sigs) == 0 {
+		return nil
+	}
+
+	// both sigs and methods are sorted by name,
+	// so we can find the intersect in a single pass
+	for _, m := range methods {
+		if m.name == sigs[0].name {
+			out = append(out, Linksym(m.isym))
+			sigs = sigs[1:]
+			if len(sigs) == 0 {
+				break
+			}
+		}
+	}
+
+	return out
+}
+
+// itabsym is used by the SSA backend
+// to convert interface calls into static calls;
+// it deliberately doesn't create any additional Nodes, Syms, etc.
+func itabsym(it *obj.LSym, offset int64) *obj.LSym {
+	var syms []*obj.LSym
+
+	// TODO: something that isn't O(itabs)?
+	// We expect this lookup to occur
+	// infrequently in practice; perhaps
+	// this is offset by O(1) itab creation.
+	for i := range itabs {
+		e := &itabs[i]
+		if e.lsym == it {
+			syms = e.entries
+			break
+		}
+	}
+	if syms == nil {
+		return nil
+	}
+
+	// keep this arithmetic in sync with *itab layout
+	methodnum := int((offset - 3*int64(Widthptr) - 8) / int64(Widthptr))
+	if methodnum >= len(syms) {
+		return nil
+	}
+	return syms[methodnum]
+}
+
 func dumptypestructs() {
 	// copy types from externdcl list to signatlist
 	for _, n := range externdcl {
@@ -1417,10 +1492,10 @@ func dumptypestructs() {
 		// }
 		o := dsymptr(i.sym, 0, dtypesym(i.itype), 0)
 		o = dsymptr(i.sym, o, dtypesym(i.t), 0)
-		o += Widthptr                          // skip link field
-		o = duint32(i.sym, o, typehash(i.t))   // copy of type hash
-		o += 4                                 // skip bad/inhash/unused fields
-		o += len(imethods(i.itype)) * Widthptr // skip fun method pointers
+		o += Widthptr                                // skip link field
+		o = duint32(i.sym, o, typehash(i.t))         // copy of type hash
+		o += 4                                       // skip bad/inhash/unused fields
+		o += len(imethods(i.itype, true)) * Widthptr // skip fun method pointers
 		// at runtime the itab will contain pointers to types, other itabs and
 		// method functions. None are allocated on heap, so we can use obj.NOPTR.
 		ggloblsym(i.sym, int32(o), int16(obj.DUPOK|obj.NOPTR))

src/cmd/compile/internal/gc/sinit.go

@@ -502,7 +502,7 @@ func staticassign(l *Node, r *Node, out *[]*Node) bool {
 		if l.Type.IsEmptyInterface() {
 			itab = typename(val.Type)
 		} else {
-			itab = itabname(val.Type, l.Type)
+			itab = itabname(val.Type, l.Type, true)
 		}
 
 		// Create a copy of l to modify while we emit data.

src/cmd/compile/internal/gc/ssa.go

@@ -4090,7 +4090,10 @@ func (s *state) dottype(n *Node, commaok bool) (res, resok *ssa.Value) {
 		targetITab = target
 	} else {
 		// Looking for pointer to itab for target type and source interface.
-		targetITab = s.expr(itabname(n.Type, n.Left.Type))
+		// Don't generate the implementation functions; there's no
+		// de-virtualization opportunity here, and we're already
+		// compiling a function.
+		targetITab = s.expr(itabname(n.Type, n.Left.Type, false))
 	}
 
 	var tmp *Node       // temporary for use with large types
@@ -4884,6 +4887,10 @@ func (e *ssaExport) SplitArray(name ssa.LocalSlot) ssa.LocalSlot {
 	return ssa.LocalSlot{N: n, Type: et, Off: name.Off}
 }
 
+func (e *ssaExport) DerefItab(it *obj.LSym, offset int64) *obj.LSym {
+	return itabsym(it, offset)
+}
+
 // namedAuto returns a new AUTO variable with the given name and type.
 // These are exposed to the debugger.
 func (e *ssaExport) namedAuto(name string, typ ssa.Type) ssa.GCNode {

src/cmd/compile/internal/gc/walk.go

@@ -386,6 +386,10 @@ func isSmallMakeSlice(n *Node) bool {
 func walkexprlist(s []*Node, init *Nodes) {
 	for i := range s {
 		s[i] = walkexpr(s[i], init)
+		if Curfn == nil {
+			println(s[i].String())
+			panic("curfn is nil")
+		}
 	}
 }
 
@@ -884,7 +888,7 @@ opswitch:
 			if n.Type.IsEmptyInterface() {
 				t = typename(n.Left.Type)
 			} else {
-				t = itabname(n.Left.Type, n.Type)
+				t = itabname(n.Left.Type, n.Type, true)
 			}
 			l := nod(OEFACE, t, n.Left)
 			l.Type = n.Type
@@ -932,7 +936,7 @@ opswitch:
 			if n.Type.IsEmptyInterface() {
 				t = typename(n.Left.Type)
 			} else {
-				t = itabname(n.Left.Type, n.Type)
+				t = itabname(n.Left.Type, n.Type, true)
 			}
 			l := nod(OEFACE, t, typecheck(nod(OADDR, value, nil), Erv))
 			l.Type = n.Type
@@ -978,7 +982,7 @@ opswitch:
 			if n.Left.Type.IsInterface() {
 				ll = append(ll, typename(n.Type))
 			} else {
-				ll = append(ll, itabname(n.Left.Type, n.Type))
+				ll = append(ll, itabname(n.Left.Type, n.Type, false))
 			}
 		}
 

src/cmd/compile/internal/ssa/config.go

@@ -121,6 +121,10 @@ type Frontend interface {
 	SplitArray(LocalSlot) LocalSlot              // array must be length 1
 	SplitInt64(LocalSlot) (LocalSlot, LocalSlot) // returns (hi, lo)
 
+	// DerefItab dereferences an itab function
+	// entry, given the byte offset of the entry.
+	DerefItab(sym *obj.LSym, offset int64) *obj.LSym
+
 	// Line returns a string describing the given position.
 	Line(src.XPos) string
 

src/cmd/compile/internal/ssa/export_test.go

@@ -84,21 +84,22 @@ func (d DummyFrontend) Warnl(_ src.XPos, msg string, args ...interface{})  { d.t
 func (d DummyFrontend) Debug_checknil() bool                               { return false }
 func (d DummyFrontend) Debug_wb() bool                                     { return false }
 
-func (d DummyFrontend) TypeBool() Type    { return TypeBool }
-func (d DummyFrontend) TypeInt8() Type    { return TypeInt8 }
-func (d DummyFrontend) TypeInt16() Type   { return TypeInt16 }
-func (d DummyFrontend) TypeInt32() Type   { return TypeInt32 }
-func (d DummyFrontend) TypeInt64() Type   { return TypeInt64 }
-func (d DummyFrontend) TypeUInt8() Type   { return TypeUInt8 }
-func (d DummyFrontend) TypeUInt16() Type  { return TypeUInt16 }
-func (d DummyFrontend) TypeUInt32() Type  { return TypeUInt32 }
-func (d DummyFrontend) TypeUInt64() Type  { return TypeUInt64 }
-func (d DummyFrontend) TypeFloat32() Type { return TypeFloat32 }
-func (d DummyFrontend) TypeFloat64() Type { return TypeFloat64 }
-func (d DummyFrontend) TypeInt() Type     { return TypeInt64 }
-func (d DummyFrontend) TypeUintptr() Type { return TypeUInt64 }
-func (d DummyFrontend) TypeString() Type  { panic("unimplemented") }
-func (d DummyFrontend) TypeBytePtr() Type { return TypeBytePtr }
+func (d DummyFrontend) TypeBool() Type                               { return TypeBool }
+func (d DummyFrontend) TypeInt8() Type                               { return TypeInt8 }
+func (d DummyFrontend) TypeInt16() Type                              { return TypeInt16 }
+func (d DummyFrontend) TypeInt32() Type                              { return TypeInt32 }
+func (d DummyFrontend) TypeInt64() Type                              { return TypeInt64 }
+func (d DummyFrontend) TypeUInt8() Type                              { return TypeUInt8 }
+func (d DummyFrontend) TypeUInt16() Type                             { return TypeUInt16 }
+func (d DummyFrontend) TypeUInt32() Type                             { return TypeUInt32 }
+func (d DummyFrontend) TypeUInt64() Type                             { return TypeUInt64 }
+func (d DummyFrontend) TypeFloat32() Type                            { return TypeFloat32 }
+func (d DummyFrontend) TypeFloat64() Type                            { return TypeFloat64 }
+func (d DummyFrontend) TypeInt() Type                                { return TypeInt64 }
+func (d DummyFrontend) TypeUintptr() Type                            { return TypeUInt64 }
+func (d DummyFrontend) TypeString() Type                             { panic("unimplemented") }
+func (d DummyFrontend) TypeBytePtr() Type                            { return TypeBytePtr }
+func (d DummyFrontend) DerefItab(sym *obj.LSym, off int64) *obj.LSym { return nil }
 
 func (d DummyFrontend) CanSSA(t Type) bool {
 	// There are no un-SSAable types in dummy land.

src/cmd/compile/internal/ssa/gen/generic.rules

@@ -1430,3 +1430,7 @@
 	&& c == config.ctxt.FixedFrameSize() + config.RegSize // offset of return value
 	&& warnRule(config.Debug_checknil() && v.Pos.Line() > 1, v, "removed nil check")
 	-> (Invalid)
+
+// De-virtualize interface calls
+(InterCall [argsize] (Load (OffPtr [off] (ITab (IMake (Addr {itab} (SB)) z))) _) mem) && devirt(v, itab, off) != nil ->
+	(StaticCall [argsize] {devirt(v, itab, off)} cleanIntercall(mem, z))

src/cmd/compile/internal/ssa/gen/rulegen.go

@@ -664,7 +664,7 @@ func parseValue(val string, arch arch, loc string) (op opData, oparch string, ty
 	}
 	if aux != "" {
 		switch op.aux {
-		case "String", "Sym", "SymOff", "SymValAndOff", "SymInt32":
+		case "String", "Sym", "SymOff", "SymValAndOff", "SymInt32", "Typ":
 		default:
 			log.Fatalf("%s: op %s %s can't have aux", loc, op.name, op.aux)
 		}