Implement PGO builds

go.mod

@@ -3,15 +3,15 @@ module github.com/tliron/puccini
 go 1.20
 
 require (
-	github.com/dop251/goja v0.0.0-20230531210528-d7324b2d74f7
+	github.com/dop251/goja v0.0.0-20230626124041-ba8a63e79201
 	github.com/fxamacker/cbor/v2 v2.4.0
-	github.com/klauspost/compress v1.16.5
+	github.com/klauspost/compress v1.16.6
 	github.com/klauspost/pgzip v1.2.6
 	github.com/segmentio/ksuid v1.0.4
 	github.com/spf13/cobra v1.7.0
 	github.com/tliron/commonjs-goja v0.1.0
 	github.com/tliron/commonlog v0.1.0
-	github.com/tliron/exturl v0.2.3
+	github.com/tliron/exturl v0.2.4
 	github.com/tliron/go-ard v0.1.1
 	github.com/tliron/kutil v0.2.5
 	github.com/tliron/yamlkeys v1.3.6
@@ -28,7 +28,7 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/docker/cli v23.0.5+incompatible // indirect
-	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/docker v23.0.5+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect

go.sum

@@ -31,15 +31,15 @@ github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo
 github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
 github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
-github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
+github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v23.0.5+incompatible h1:DaxtlTJjFSnLOXVNUBU1+6kXGz2lpDoEAH6QoxaSg8k=
 github.com/docker/docker v23.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
 github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
 github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
-github.com/dop251/goja v0.0.0-20230531210528-d7324b2d74f7 h1:cVGkvrdHgyBkYeB6kMCaF5j2d9Bg4trgbIpcUrKrvk4=
-github.com/dop251/goja v0.0.0-20230531210528-d7324b2d74f7/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
+github.com/dop251/goja v0.0.0-20230626124041-ba8a63e79201 h1:+9NRIliCUhliHMCixEO0mcXmrv3HYwxs9oxM1Z+qnYM=
+github.com/dop251/goja v0.0.0-20230626124041-ba8a63e79201/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
 github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
 github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
 github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
@@ -83,8 +83,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
-github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
-github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
+github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kortschak/utter v1.5.0 h1:1vHGHPZmJ6zU5XbfllIAG3eQBoHT97ePrZJ+pT3RoiQ=
@@ -162,8 +162,8 @@ github.com/tliron/commonjs-goja v0.1.0 h1:BF1NitaiHl2RLxqjp9lGNXx2eiJtTKKk3UFpSE
 github.com/tliron/commonjs-goja v0.1.0/go.mod h1:mgGnXjwqvLSzMqsfLpwFGKS5qz40b66rE97IWHLQUSA=
 github.com/tliron/commonlog v0.1.0 h1:EsNHnG/oPSJ8BqTjIRxLc/lmurz5cyUuoKe7CabmHfg=
 github.com/tliron/commonlog v0.1.0/go.mod h1:aF5mXemxKqtWPIvsexXjgKXWLkXnBUt33mqek5E0UJc=
-github.com/tliron/exturl v0.2.3 h1:nns3PHkBG1WKMRXlBlhTjcW+/OmgGNXPJku9zwcO3TU=
-github.com/tliron/exturl v0.2.3/go.mod h1:lMm4XTcIMul+BIMein+aeDEkdpkcHMyJyWL4Sr3J/e4=
+github.com/tliron/exturl v0.2.4 h1:/7lvJUrfJgxNyzMaeS6lKXj6NbDJf3BenoncgVzZChY=
+github.com/tliron/exturl v0.2.4/go.mod h1:lMm4XTcIMul+BIMein+aeDEkdpkcHMyJyWL4Sr3J/e4=
 github.com/tliron/go-ard v0.1.1 h1:Z0JD63j387RmjiJNuMHEmj6+ht3Aj0/tY3sSbv/MzRU=
 github.com/tliron/go-ard v0.1.1/go.mod h1:b8BG7fxGd/trwh9zD6FH797xgRLa9g6SN+sR4xrkzEA=
 github.com/tliron/kutil v0.2.5 h1:ec5rOYpIcEimWX3/1G05EYAtMNqENjAn9FDOiCWquSo=

puccini-clout/commands/root.go

@@ -1,6 +1,9 @@
 package commands
 
 import (
+	"os"
+	"runtime/pprof"
+
 	"github.com/spf13/cobra"
 	"github.com/tliron/commonlog"
 	"github.com/tliron/kutil/terminal"
@@ -14,6 +17,7 @@ var inputFormat string
 var colorize string
 var strict bool
 var pretty bool
+var cpuProfilePath string
 
 func init() {
 	rootCommand.PersistentFlags().BoolVarP(&terminal.Quiet, "quiet", "q", false, "suppress output")
@@ -24,6 +28,7 @@ func init() {
 	rootCommand.PersistentFlags().StringVarP(&colorize, "colorize", "z", "true", "colorize output (boolean or \"force\")")
 	rootCommand.PersistentFlags().BoolVarP(&strict, "strict", "y", false, "strict output (for \"yaml\" format only)")
 	rootCommand.PersistentFlags().BoolVarP(&pretty, "pretty", "p", true, "prettify output")
+	rootCommand.PersistentFlags().StringVarP(&cpuProfilePath, "cpu-profile", "", "", "CPU profile file path")
 }
 
 var rootCommand = &cobra.Command{
@@ -35,6 +40,7 @@ var rootCommand = &cobra.Command{
 		if cleanup != nil {
 			util.OnExitError(cleanup)
 		}
+
 		if logTo == "" {
 			if terminal.Quiet {
 				verbose = -4
@@ -43,6 +49,14 @@ var rootCommand = &cobra.Command{
 		} else {
 			commonlog.Configure(verbose, &logTo)
 		}
+
+		if cpuProfilePath != "" {
+			cpuProfile, err := os.Create(cpuProfilePath)
+			util.FailOnError(err)
+			err = pprof.StartCPUProfile(cpuProfile)
+			util.FailOnError(err)
+			util.OnExit(pprof.StopCPUProfile)
+		}
 	},
 }
 

puccini-csar/commands/root.go

@@ -1,6 +1,9 @@
 package commands
 
 import (
+	"os"
+	"runtime/pprof"
+
 	"github.com/spf13/cobra"
 	"github.com/tliron/commonlog"
 	"github.com/tliron/kutil/terminal"
@@ -9,11 +12,13 @@ import (
 
 var logTo string
 var verbose int
+var cpuProfilePath string
 
 func init() {
 	rootCommand.PersistentFlags().BoolVarP(&terminal.Quiet, "quiet", "q", false, "suppress output")
 	rootCommand.PersistentFlags().StringVarP(&logTo, "log", "l", "", "log to file (defaults to stderr)")
 	rootCommand.PersistentFlags().CountVarP(&verbose, "verbose", "v", "add a log verbosity level (can be used twice)")
+	rootCommand.PersistentFlags().StringVarP(&cpuProfilePath, "cpu-profile", "", "", "CPU profile file path")
 }
 
 var rootCommand = &cobra.Command{
@@ -25,6 +30,7 @@ var rootCommand = &cobra.Command{
 		if cleanup != nil {
 			util.OnExitError(cleanup)
 		}
+
 		if logTo == "" {
 			if terminal.Quiet {
 				verbose = -4
@@ -33,6 +39,14 @@ var rootCommand = &cobra.Command{
 		} else {
 			commonlog.Configure(verbose, &logTo)
 		}
+
+		if cpuProfilePath != "" {
+			cpuProfile, err := os.Create(cpuProfilePath)
+			util.FailOnError(err)
+			err = pprof.StartCPUProfile(cpuProfile)
+			util.FailOnError(err)
+			util.OnExit(pprof.StopCPUProfile)
+		}
 	},
 }
 

puccini-tosca/commands/root.go

@@ -1,6 +1,9 @@
 package commands
 
 import (
+	"os"
+	"runtime/pprof"
+
 	"github.com/spf13/cobra"
 	"github.com/tliron/commonlog"
 	"github.com/tliron/kutil/terminal"
@@ -13,6 +16,7 @@ var format string
 var colorize string
 var strict bool
 var pretty bool
+var cpuProfilePath string
 
 func init() {
 	rootCommand.PersistentFlags().BoolVarP(&terminal.Quiet, "quiet", "q", false, "suppress output")
@@ -22,6 +26,7 @@ func init() {
 	rootCommand.PersistentFlags().StringVarP(&colorize, "colorize", "z", "true", "colorize output (boolean or \"force\")")
 	rootCommand.PersistentFlags().BoolVarP(&strict, "strict", "y", false, "strict output (for \"yaml\" format only)")
 	rootCommand.PersistentFlags().BoolVarP(&pretty, "pretty", "p", true, "prettify output")
+	rootCommand.PersistentFlags().StringVarP(&cpuProfilePath, "cpu-profile", "", "", "CPU profile file path")
 }
 
 var rootCommand = &cobra.Command{
@@ -33,6 +38,7 @@ var rootCommand = &cobra.Command{
 		if cleanup != nil {
 			util.OnExitError(cleanup)
 		}
+
 		if logTo == "" {
 			if terminal.Quiet {
 				verbose = -4
@@ -41,6 +47,14 @@ var rootCommand = &cobra.Command{
 		} else {
 			commonlog.Configure(verbose, &logTo)
 		}
+
+		if cpuProfilePath != "" {
+			cpuProfile, err := os.Create(cpuProfilePath)
+			util.FailOnError(err)
+			err = pprof.StartCPUProfile(cpuProfile)
+			util.FailOnError(err)
+			util.OnExit(pprof.StopCPUProfile)
+		}
 	},
 }
 

scripts/build

@@ -14,12 +14,17 @@ function build () {
 	local TOOL=$1
 	pushd "$ROOT/$TOOL" > /dev/null
 	CGO_ENABLED=0 go install \
+		-pgo=auto \
 		-ldflags " \
 			-X 'github.com/tliron/kutil/version.GitVersion=$VERSION' \
 			-X 'github.com/tliron/kutil/version.GitRevision=$REVISION' \
 			-X 'github.com/tliron/kutil/version.Timestamp=$TIMESTAMP'"
+	if [ -f default.pgo ]; then
+		m "built ${GOPATH//\\/\\\\}/bin/$TOOL (pgo)"
+	else
+		m "built ${GOPATH//\\/\\\\}/bin/$TOOL"
+	fi
 	popd > /dev/null
-	m "built ${GOPATH//\\/\\\\}/bin/$TOOL"
 }
 
 build puccini-tosca

scripts/build-library

@@ -15,6 +15,7 @@ function build () {
 	pushd "$ROOT/$TOOL" > /dev/null
 	go build \
 		-buildmode=c-shared \
+		-pgo=auto \
 		-o="$ROOT/dist/libpuccini.so" \
 		-ldflags " \
 			-X 'github.com/tliron/kutil/version.GitVersion=$VERSION' \

scripts/build-profiles

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -e
+
+HERE=$(dirname "$(readlink --canonicalize "$BASH_SOURCE")")
+. "$HERE/_env"
+
+rm --force "$ROOT/puccini-tosca/default.pgo"
+rm --force "$ROOT/puccini-clout/default.pgo"
+rm --force "$ROOT/puccini-csar/default.pgo"
+rm --force "$ROOT/library/default.pgo"
+
+"$HERE/test"
+
+m "creating $ROOT/puccini-csar/default.pgo"
+
+mkdir --parents "$ROOT/dist"
+
+puccini-csar create "$ROOT/dist/cloud.tar.gz" "$ROOT/examples/csar" \
+    --entry-definitions=main.yaml \
+    --other-definitions='other 1.yaml' \
+    --other-definitions='other 2.yaml' \
+	--cpu-profile="$ROOT/puccini-csar/default.pgo" > /dev/null 2>&1
+
+m "creating $ROOT/puccini-tosca/default.pgo"
+
+. "$HERE/_start-http-server"
+
+puccini-tosca compile "http://localhost:8000/cloud.tar.gz" --coerce \
+	--cpu-profile="$ROOT/puccini-tosca/default.pgo" > /dev/null 2>&1
+
+cp "$ROOT/puccini-tosca/default.pgo" "$ROOT/library/default.pgo"
+
+m "creating $ROOT/puccini-clout/default.pgo"
+
+puccini-tosca compile "http://localhost:8000/cloud.tar.gz" | \
+puccini-clout scriptlet exec tosca.resolve \
+	--cpu-profile="$ROOT/puccini-clout/default.pgo" > /dev/null 2>&1
+
+"$HERE/test"