Merge pull request #4279 from jedevc/containerd-runtime-option-fixups

Add options to specify containerd runtime (alternative)

cmd/buildkitd/config/config.go

@@ -112,6 +112,7 @@ type ContainerdConfig struct {
 	Labels    map[string]string `toml:"labels"`
 	Platforms []string          `toml:"platforms"`
 	Namespace string            `toml:"namespace"`
+	Runtime   ContainerdRuntime `toml:"runtime"`
 	GCConfig
 	NetworkConfig
 	Snapshotter string `toml:"snapshotter"`
@@ -128,6 +129,11 @@ type ContainerdConfig struct {
 	Rootless bool `toml:"rootless"`
 }
 
+type ContainerdRuntime struct {
+	Name    string                 `toml:"name"`
+	Options map[string]interface{} `toml:"options"`
+}
+
 type GCPolicy struct {
 	All          bool      `toml:"all"`
 	KeepBytes    DiskSpace `toml:"keepBytes"`

cmd/buildkitd/config/load_test.go

@@ -42,6 +42,9 @@ foo="bar"
 namespace="non-default"
 platforms=["linux/amd64"]
 address="containerd.sock"
+[worker.containerd.runtime]
+name="exotic"
+options.foo="bar"
 [[worker.containerd.gcpolicy]]
 all=true
 filters=["foo==bar"]
@@ -103,6 +106,8 @@ searchDomains=["example.com"]
 
 	require.Equal(t, 0, len(cfg.Workers.OCI.GCPolicy))
 	require.Equal(t, "non-default", cfg.Workers.Containerd.Namespace)
+	require.Equal(t, "exotic", cfg.Workers.Containerd.Runtime.Name)
+	require.Equal(t, "bar", cfg.Workers.Containerd.Runtime.Options["foo"])
 	require.Equal(t, 3, len(cfg.Workers.Containerd.GCPolicy))
 
 	require.Nil(t, cfg.Workers.Containerd.GC)

cmd/buildkitd/main_containerd_worker.go

@@ -6,19 +6,25 @@ package main
 import (
 	"context"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
 	ctd "github.com/containerd/containerd"
+	"github.com/containerd/containerd/defaults"
+	runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1"
 	"github.com/containerd/containerd/pkg/userns"
+	"github.com/containerd/containerd/plugin"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/moby/buildkit/cmd/buildkitd/config"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/network/cniprovider"
 	"github.com/moby/buildkit/util/network/netproviders"
 	"github.com/moby/buildkit/worker"
 	"github.com/moby/buildkit/worker/base"
 	"github.com/moby/buildkit/worker/containerd"
+	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 	"golang.org/x/sync/semaphore"
@@ -46,6 +52,15 @@ func init() {
 		defaultConf.Workers.Containerd.Namespace = defaultContainerdNamespace
 	}
 
+	if defaultConf.Workers.Containerd.Runtime.Name == "" {
+		if runtime.GOOS == "freebsd" {
+			// TODO: this can be removed once containerd/containerd#8964 is included
+			defaultConf.Workers.Containerd.Runtime.Name = "wtf.sbk.runj.v1"
+		} else {
+			defaultConf.Workers.Containerd.Runtime.Name = defaults.DefaultRuntime
+		}
+	}
+
 	flags := []cli.Flag{
 		cli.StringFlag{
 			Name:  "containerd-worker",
@@ -74,6 +89,12 @@ func init() {
 			Value:  defaultConf.Workers.Containerd.Namespace,
 			Hidden: true,
 		},
+		cli.StringFlag{
+			Name:   "containerd-worker-runtime",
+			Usage:  "override containerd runtime",
+			Value:  defaultConf.Workers.Containerd.Runtime.Name,
+			Hidden: true,
+		},
 		cli.StringFlag{
 			Name:  "containerd-worker-net",
 			Usage: "worker network type (auto, cni or host)",
@@ -202,6 +223,12 @@ func applyContainerdFlags(c *cli.Context, cfg *config.Config) error {
 		cfg.Workers.Containerd.Namespace = c.GlobalString("containerd-worker-namespace")
 	}
 
+	if c.GlobalIsSet("containerd-worker-runtime") || cfg.Workers.Containerd.Runtime.Name == "" {
+		cfg.Workers.Containerd.Runtime = config.ContainerdRuntime{
+			Name: c.GlobalString("containerd-worker-runtime"),
+		}
+	}
+
 	if c.GlobalIsSet("containerd-worker-gc") {
 		v := c.GlobalBool("containerd-worker-gc")
 		cfg.Workers.Containerd.GC = &v
@@ -275,7 +302,26 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([
 	if cfg.Snapshotter != "" {
 		snapshotter = cfg.Snapshotter
 	}
-	opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Rootless, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, common.config.Workers.Containerd.SELinux, parallelismSem, common.traceSocket, ctd.WithTimeout(60*time.Second))
+
+	var runtime *containerd.RuntimeInfo
+	if cfg.Runtime.Name != "" {
+		opts := getRuntimeOptionsType(cfg.Runtime.Name)
+
+		t, err := toml.TreeFromMap(cfg.Runtime.Options)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse runtime options config")
+		}
+		err = t.Unmarshal(opts)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse runtime options config")
+		}
+
+		runtime = &containerd.RuntimeInfo{
+			Name:    cfg.Runtime.Name,
+			Options: opts,
+		}
+	}
+	opt, err := containerd.NewWorkerOpt(common.config.Root, cfg.Address, snapshotter, cfg.Namespace, cfg.Rootless, cfg.Labels, dns, nc, common.config.Workers.Containerd.ApparmorProfile, common.config.Workers.Containerd.SELinux, parallelismSem, common.traceSocket, runtime, ctd.WithTimeout(60*time.Second))
 	if err != nil {
 		return nil, err
 	}
@@ -320,3 +366,13 @@ func validContainerdSocket(cfg config.ContainerdConfig) bool {
 	}
 	return true
 }
+
+// getRuntimeOptionsType gets empty runtime options by the runtime type name.
+func getRuntimeOptionsType(t string) interface{} {
+	switch t {
+	case plugin.RuntimeRuncV2:
+		return &runcoptions.Options{}
+	default:
+		return &runtimeoptions.Options{}
+	}
+}

docs/buildkitd.toml.md

@@ -106,6 +106,11 @@ insecure-entitlements = [ "network.host", "security.insecure" ]
   [worker.containerd.labels]
     "foo" = "bar"
 
+  # configure the containerd runtime
+  [worker.containerd.runtime]
+    runtime = "io.containerd.runc.v2"
+    options = { BinaryName = "runc" }
+
   [[worker.containerd.gcpolicy]]
     keepBytes = 512000000
     keepDuration = 172800

executor/containerdexecutor/executor.go

@@ -38,6 +38,7 @@ type containerdExecutor struct {
 	selinux          bool
 	traceSocket      string
 	rootless         bool
+	runtime          *RuntimeInfo
 }
 
 // OnCreateRuntimer provides an alternative to OCI hooks for applying network
@@ -53,8 +54,13 @@ type OnCreateRuntimer interface {
 	OnCreateRuntime(pid uint32) error
 }
 
+type RuntimeInfo struct {
+	Name    string
+	Options any
+}
+
 // New creates a new executor backed by connection to containerd API
-func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, selinux bool, traceSocket string, rootless bool) executor.Executor {
+func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig, apparmorProfile string, selinux bool, traceSocket string, rootless bool, runtime *RuntimeInfo) executor.Executor {
 	// clean up old hosts/resolv.conf file. ignore errors
 	os.RemoveAll(filepath.Join(root, "hosts"))
 	os.RemoveAll(filepath.Join(root, "resolv.conf"))
@@ -70,6 +76,7 @@ func New(client *containerd.Client, root, cgroup string, networkProviders map[pb
 		selinux:          selinux,
 		traceSocket:      traceSocket,
 		rootless:         rootless,
+		runtime:          runtime,
 	}
 }
 
@@ -145,9 +152,13 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M
 		defer releaseSpec()
 	}
 
-	container, err := w.client.NewContainer(ctx, id,
+	opts := []containerd.NewContainerOpts{
 		containerd.WithSpec(spec),
-	)
+	}
+	if w.runtime != nil {
+		opts = append(opts, containerd.WithRuntime(w.runtime.Name, w.runtime.Options))
+	}
+	container, err := w.client.NewContainer(ctx, id, opts...)
 	if err != nil {
 		return nil, err
 	}