Skip to content

Commit

Permalink
Sync with upstream. Reduce duplication when working with references i…
Browse files Browse the repository at this point in the history 
…n solver

Signed-off-by: a-palchikov <[email protected]>
  • Loading branch information
@a-palchikov
a-palchikov committed Oct 6, 2022
1 parent a4e8174 commit a8ec866
Show file tree
Hide file tree
Showing 7 changed files with 86 additions and 87 deletions.
6 changes: 3 additions & 3 deletions exporter/containerimage/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,11 +196,11 @@ func (e *imageExporterInstance) Config() exporter.Config {
}
}

func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, sessionID string) (map[string]string, error) {
func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source, sessionID string) (map[string]string, error) {
return e.ExportImage(ctx, src, exptypes.InlineCache{}, sessionID)
}

func (e *imageExporterInstance) ExportImage(ctx context.Context, src exporter.Source, inlineCache exptypes.InlineCache, sessionID string) (map[string]string, error) {
func (e *imageExporterInstance) ExportImage(ctx context.Context, src *exporter.Source, inlineCache exptypes.InlineCache, sessionID string) (map[string]string, error) {
meta := make(map[string][]byte)
for k, v := range src.Metadata {
meta[k] = v
Expand Down Expand Up @@ -364,7 +364,7 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou
return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, annotations)
}

func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src exporter.Source, s session.Group) (err0 error) {
func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src *exporter.Source, s session.Group) (err0 error) {
unpackDone := progress.OneOff(ctx, "unpacking to "+img.Name)
defer func() {
unpackDone(err0)
Expand Down
2 changes: 1 addition & 1 deletion exporter/containerimage/writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ type ImageWriter struct {
opt WriterOpt
}

func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, sessionID string, opts *ImageCommitOpts) (*ocispecs.Descriptor, error) {
func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, sessionID string, opts *ImageCommitOpts) (*ocispecs.Descriptor, error) {
platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey]

if len(inp.Refs) > 0 && !ok {
Expand Down
2 changes: 1 addition & 1 deletion exporter/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ type Exporter interface {
type ExporterInstance interface {
Name() string
Config() Config
Export(ctx context.Context, src Source, sessionID string) (map[string]string, error)
Export(ctx context.Context, src *Source, sessionID string) (map[string]string, error)
}

type Config struct {
Expand Down
8 changes: 3 additions & 5 deletions exporter/oci/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,11 @@ func (e *imageExporterInstance) Config() exporter.Config {
}
}

func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, sessionID string) (map[string]string, error) {
func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source, sessionID string) (map[string]string, error) {
return e.ExportImage(ctx, src, exptypes.InlineCache{}, sessionID)
}

func (e *imageExporterInstance) ExportImage(ctx context.Context, src exporter.Source, inlineCache exptypes.InlineCache, sessionID string) (map[string]string, error) {
func (e *imageExporterInstance) ExportImage(ctx context.Context, src *exporter.Source, inlineCache exptypes.InlineCache, sessionID string) (map[string]string, error) {
if e.opt.Variant == VariantDocker && len(src.Refs) > 0 {
return nil, errors.Errorf("docker exporter does not currently support exporting manifest lists")
}
Expand All @@ -111,14 +111,12 @@ func (e *imageExporterInstance) ExportImage(ctx context.Context, src exporter.So
}

opts := e.opts
as, _, err := containerimage.ParseAnnotations(metadata)
as, _, err := containerimage.ParseAnnotations(meta)
if err != nil {
return nil, err
}
opts.AddAnnotations(as)

opts := e.opts
as, _, err := containerimage.ParseAnnotations(src.Metadata)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion exporter/tar/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (e *localExporterInstance) Config() exporter.Config {
return exporter.Config{}
}

func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source, sessionID string) (map[string]string, error) {
func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source, sessionID string) (map[string]string, error) {
var defers []func()

defer func() {
Expand Down
136 changes: 60 additions & 76 deletions solver/llbsolver/solver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,52 +192,41 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
}
}

var exporterResponse map[string]string
for _, e := range exp.Exporters {
cached, err := result.ConvertResult(res, func(res solver.ResultProxy) (solver.CachedResult, error) {
return res.Result(ctx)
})
if err != nil {
return nil, err
}
cached, err := result.ConvertResult(res, func(res solver.ResultProxy) (solver.CachedResult, error) {
return res.Result(ctx)
})
if err != nil {
return nil, err
}

inp, err := result.ConvertResult(cached, func(res solver.CachedResult) (cache.ImmutableRef, error) {
workerRef, ok := res.Sys().(*worker.WorkerRef)
if !ok {
return nil, errors.Errorf("invalid reference: %T", r.Sys())
}
return workerRef.ImmutableRef, nil
})
if err != nil {
return nil, err
inp, err := result.ConvertResult(cached, func(res solver.CachedResult) (cache.ImmutableRef, error) {
workerRef, ok := res.Sys().(*worker.WorkerRef)
if !ok {
return nil, errors.Errorf("invalid reference: %T", res.Sys())
}
return workerRef.ImmutableRef, nil
})
if err != nil {
return nil, err
}

cacheExporter := exp.CacheExporter
if _, ok := asInlineCache(cacheExporter); ok {
exp.CacheExporter = nil
}
exporterResponse, err = s.runExporters(ctx, exp.Exporters, cacheExporter, cached, inp, j)
if err != nil {
return nil, err
}
var inlineCacheExporter inlineCacheExporter
var ok bool
if inlineCacheExporter, ok = asInlineCache(exp.CacheExporter); ok {
exp.CacheExporter = nil
}
exporterResponse, err := s.runExporters(ctx, exp.Exporters, inlineCacheExporter, j, cached, inp)
if err != nil {
return nil, err
}

g := session.NewGroup(j.SessionID)
var cacheExporterResponse map[string]string
if e := exp.CacheExporter; e != nil {
if err := inBuilderContext(ctx, j, "exporting cache", j.SessionID+"-cache", func(ctx context.Context, _ session.Group) error {
prepareDone := progress.OneOff(ctx, "preparing build cache for export")
if err := res.EachRef(func(res solver.ResultProxy) error {
r, err := res.Result(ctx)
if err != nil {
return err
}

workerRef, ok := r.Sys().(*worker.WorkerRef)
if !ok {
return errors.Errorf("invalid reference: %T", r.Sys())
}
ctx = withDescHandlerCacheOpts(ctx, workerRef.ImmutableRef)
if err := result.EachRef(cached, inp, func(r solver.CachedResult, ir cache.ImmutableRef) error {
ctx = withDescHandlerCacheOpts(ctx, ir)

// Configure compression
compressionConfig := e.Config().Compression
Expand Down Expand Up @@ -293,58 +282,56 @@ func imageExporters(exporters []exporter.ExporterInstance) (result []imageExport
return result
}

func (s *Solver) runExporters(ctx context.Context, exporters []exporter.ExporterInstance, cacheExporter remotecache.Exporter, cached *result.Result[solver.CachedResult], inp exporter.Source, job *solver.Job) (exporterResponse map[string]string, err error) {
func (s *Solver) runExporters(ctx context.Context, exporters []exporter.ExporterInstance, inlineCacheExporter inlineCacheExporter, job *solver.Job, cached *result.Result[solver.CachedResult], inp *result.Result[cache.ImmutableRef]) (exporterResponse map[string]string, err error) {
eg, ctx := errgroup.WithContext(ctx)
sessionID := job.SessionID
for _, exp := range exporters {
exp := exp
eg.Go(func() error {
if imageExporter, ok := asImageExporter(exp); ok {
var inlineCache exptypes.InlineCache
if _, ok := asInlineCache(cacheExporter); ok {
ic, err := exportInlineCache(ctx, imageExporter.Config().Compression, cacheExporter, cached, job, sessionID)
if err != nil {
return err
resps := make([]map[string]string, len(exporters))
for i, exp := range exporters {
func(i int, exp exporter.ExporterInstance) {
eg.Go(func() error {
if imageExporter, ok := asImageExporter(exp); ok {
var inlineCache exptypes.InlineCache
if inlineCacheExporter != nil {
ic, err := exportInlineCache(ctx, imageExporter.Config().Compression, inlineCacheExporter, cached, job, sessionID)
if err != nil {
return err
}
inlineCache = *ic
}
inlineCache = *ic
return inBuilderContext(ctx, job, exp.Name(), job.SessionID+"-export", func(ctx context.Context, _ session.Group) (err error) {
resps[i], err = imageExporter.ExportImage(ctx, inp, inlineCache, sessionID)
if err != nil {
return err
}
return nil
})
}
return inBuilderContext(ctx, job, exp.Name(), job.SessionID+"-export", func(ctx context.Context, _ session.Group) error {
resp, err := imageExporter.ExportImage(ctx, inp, inlineCache, sessionID)
resps[i], err = exp.Export(ctx, inp, sessionID)
if err != nil {
return err
}
for k, v := range resp {
if exporterResponse == nil {
exporterResponse = make(map[string]string)
}
exporterResponse[k] = v
}
return nil
})
}
return inBuilderContext(ctx, job, exp.Name(), job.SessionID+"-export", func(ctx context.Context, _ session.Group) error {
resp, err := exp.Export(ctx, inp, sessionID)
if err != nil {
return err
}
for k, v := range resp {
if exporterResponse == nil {
exporterResponse = make(map[string]string)
}
exporterResponse[k] = v
}
return nil
})
})
}(i, exp)
}
if err := eg.Wait(); err != nil {
return nil, err
}

for _, resp := range resps {
for k, v := range resp {
if exporterResponse == nil {
exporterResponse = make(map[string]string)
}
exporterResponse[k] = v
}
}
return exporterResponse, nil
}

func exportInlineCache(ctx context.Context, compression compression.Config, e remotecache.Exporter, cached *result.Result[solver.CachedResult], job *solver.Job, sessionID string) (*exptypes.InlineCache, error) {
func exportInlineCache(ctx context.Context, compression compression.Config, e inlineCacheExporter, cached *result.Result[solver.CachedResult], job *solver.Job, sessionID string) (*exptypes.InlineCache, error) {
result := &exptypes.InlineCache{}
if err := inBuilderContext(ctx, job, "preparing layers for inline cache", job.SessionID+"-cache-inline", func(ctx context.Context, _ session.Group) error {
if cached.Ref != nil {
Expand Down Expand Up @@ -374,7 +361,7 @@ func exportInlineCache(ctx context.Context, compression compression.Config, e re

type imageExporterInstance interface {
exporter.ExporterInstance
ExportImage(ctx context.Context, src exporter.Source, cache exptypes.InlineCache, sessionID string) (map[string]string, error)
ExportImage(ctx context.Context, src *exporter.Source, cache exptypes.InlineCache, sessionID string) (map[string]string, error)
}

func asImageExporter(e exporter.ExporterInstance) (imageExporterInstance, bool) {
Expand All @@ -383,6 +370,7 @@ func asImageExporter(e exporter.ExporterInstance) (imageExporterInstance, bool)
}

type inlineCacheExporter interface {
solver.CacheExporterTarget
ExportForLayers(context.Context, []digest.Digest) ([]byte, error)
}

Expand All @@ -391,11 +379,7 @@ func asInlineCache(e remotecache.Exporter) (inlineCacheExporter, bool) {
return ie, ok
}

func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedResult, compressionopt compression.Config, g session.Group) ([]byte, error) {
ie, ok := asInlineCache(e)
if !ok {
return nil, nil
}
func inlineCache(ctx context.Context, ie inlineCacheExporter, res solver.CachedResult, compressionopt compression.Config, g session.Group) ([]byte, error) {
workerRef, ok := res.Sys().(*worker.WorkerRef)
if !ok {
return nil, errors.Errorf("invalid reference: %T", res.Sys())
Expand All @@ -414,7 +398,7 @@ func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedR

ctx = withDescHandlerCacheOpts(ctx, workerRef.ImmutableRef)
refCfg := cacheconfig.RefConfig{Compression: compressionopt}
if _, err := res.CacheKeys()[0].Exporter.ExportTo(ctx, e, solver.CacheExportOpt{
if _, err := res.CacheKeys()[0].Exporter.ExportTo(ctx, ie, solver.CacheExportOpt{
ResolveRemotes: workerRefResolver(refCfg, true, g), // load as many compression blobs as possible
Mode: solver.CacheExportModeMin,
Session: g,
Expand Down
17 changes: 17 additions & 0 deletions solver/result/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,23 @@ func (r *Result[T]) EachRef(fn func(T) error) (err error) {
return err
}

// EachRef iterates over references in both a and b.
// a and b are assumed to be of the same size and map their references
// to the same set of keys
func EachRef[U any, V any](a *Result[U], b *Result[V], fn func(U, V) error) (err error) {
if reflect.ValueOf(a.Ref).IsValid() && reflect.ValueOf(b.Ref).IsValid() {
err = fn(a.Ref, b.Ref)
}
for k, r := range a.Refs {
if reflect.ValueOf(r).IsValid() && reflect.ValueOf(b.Refs[k]).IsValid() {
if err1 := fn(r, b.Refs[k]); err1 != nil && err == nil {
err = err1
}
}
}
return err
}

func ConvertResult[U any, V any](r *Result[U], fn func(U) (V, error)) (*Result[V], error) {
r2 := &Result[V]{}
var err error
Expand Down

0 comments on commit a8ec866

Please sign in to comment.