Skip to content

Commit

Permalink
fix: worker pull usage of ram (#5097)
Browse files Browse the repository at this point in the history
  • Loading branch information
@richardlt
richardlt authored Apr 22, 2020
1 parent b52a3ae commit 724564e
Show file tree
Hide file tree
Showing 15 changed files with 506 additions and 80 deletions.
2 changes: 1 addition & 1 deletion engine/api/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ func (r *Router) handle(uri string, scope HandlerScope, handlers ...*service.Han
ctx, err = m(ctx, responseWriter, req, rc)
if err != nil {
observability.Record(r.Background, Errors, 1)
service.WriteError(ctx, w, req, err)
service.WriteError(ctx, responseWriter, req, err)
deferFunc(ctx)
return
}
Expand Down
11 changes: 7 additions & 4 deletions engine/worker/cmd_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func cmdCachePush() *cobra.Command {
Inside a project, you can create a cache from your worker with a tag (useful for vendors for example)
worker cache push <tagValue> dir/file
You can use you storage integration:
You can use you storage integration:
worker cache push --destination=MyStorageIntegration <tagValue> dir/file
`,
Example: "worker cache push {{.cds.workflow}}-{{.cds.version}} ./pathToUpload",
Expand Down Expand Up @@ -111,7 +111,7 @@ func cachePushCmd() func(cmd *cobra.Command, args []string) {
}

c := sdk.Cache{
Tag: args[0],
Tag: base64.RawURLEncoding.EncodeToString([]byte(args[0])),
Files: files,
WorkingDirectory: cwd,
IntegrationName: cmdStorageIntegrationName,
Expand All @@ -125,7 +125,7 @@ func cachePushCmd() func(cmd *cobra.Command, args []string) {
fmt.Printf("Worker cache push in progress... (tag: %s)\n", args[0])
req, errRequest := http.NewRequest(
"POST",
fmt.Sprintf("http://127.0.0.1:%d/cache/%s/push", port, base64.RawURLEncoding.EncodeToString([]byte(args[0]))),
fmt.Sprintf("http://127.0.0.1:%d/cache/push", port),
bytes.NewReader(data),
)
if errRequest != nil {
Expand Down Expand Up @@ -209,7 +209,10 @@ func cachePullCmd() func(cmd *cobra.Command, args []string) {
fmt.Printf("Worker cache pull in progress... (tag: %s)\n", args[0])
req, errRequest := http.NewRequest(
"GET",
fmt.Sprintf("http://127.0.0.1:%d/cache/%s/pull?path=%s&integration=%s", port, base64.RawURLEncoding.EncodeToString([]byte(args[0])), url.QueryEscape(dir), url.QueryEscape(cmdStorageIntegrationName)),
fmt.Sprintf("http://127.0.0.1:%d/cache/%s/pull?path=%s&integration=%s", port,
base64.RawURLEncoding.EncodeToString([]byte(args[0])),
url.QueryEscape(dir),
url.QueryEscape(cmdStorageIntegrationName)),
nil,
)
if errRequest != nil {
Expand Down
8 changes: 5 additions & 3 deletions engine/worker/internal/action/builtin_serve_static_files.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package action

import (
"bytes"
"context"
"errors"
"fmt"
Expand Down Expand Up @@ -79,16 +80,17 @@ func RunServeStaticFiles(ctx context.Context, wk workerruntime.Runtime, a sdk.Ac
}

wk.SendLog(ctx, workerruntime.LevelInfo, "Fetching files in progress...")
file, _, err := sdk.CreateTarFromPaths(aferoFS, path, filesPath, &sdk.TarOptions{TrimDirName: filepath.Dir(path)})
if err != nil {

buf := new(bytes.Buffer)
if err := sdk.CreateTarFromPaths(aferoFS, path, filesPath, buf, &sdk.TarOptions{TrimDirName: filepath.Dir(path)}); err != nil {
return res, fmt.Errorf("cannot tar files: %v", err)
}

integrationName := sdk.DefaultIfEmptyStorage(strings.TrimSpace(sdk.ParameterValue(a.Parameters, "destination")))
projectKey := sdk.ParameterValue(wk.Parameters(), "cds.project")

wk.SendLog(ctx, workerruntime.LevelInfo, fmt.Sprintf(`Upload and serving files in progress... with entrypoint "%s"`, entrypoint.Value))
publicURL, _, _, err := wk.Client().QueueStaticFilesUpload(ctx, projectKey, integrationName, jobID, name.Value, entrypoint.Value, staticKey, file)
publicURL, _, _, err := wk.Client().QueueStaticFilesUpload(ctx, projectKey, integrationName, jobID, name.Value, entrypoint.Value, staticKey, buf)
if err != nil {
return res, fmt.Errorf("Cannot upload static files: %v", err)
}
Expand Down
98 changes: 71 additions & 27 deletions engine/worker/internal/handler_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,21 @@ import (
"github.com/gorilla/mux"
"github.com/spf13/afero"

"github.com/ovh/cds/engine/worker/pkg/workerruntime"
"github.com/ovh/cds/sdk"
"github.com/ovh/cds/sdk/log"
)

func cachePushHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
// Get body
data, errRead := ioutil.ReadAll(r.Body)
if errRead != nil {
errRead = sdk.Error{
Message: "worker cache push > Cannot read body : " + errRead.Error(),
data, err := ioutil.ReadAll(r.Body)
if err != nil {
err = sdk.Error{
Message: "worker cache push > Cannot read body : " + err.Error(),
Status: http.StatusInternalServerError,
}
log.Error(ctx, "%v", errRead)
writeError(w, r, errRead)
log.Error(ctx, "%v", err)
writeError(w, r, err)
return
}

Expand All @@ -45,38 +44,77 @@ func cachePushHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc {
return
}

res, size, errTar := sdk.CreateTarFromPaths(afero.NewOsFs(), c.WorkingDirectory, c.Files, nil)
if errTar != nil {
errTar = sdk.Error{
Message: fmt.Sprintf("worker cache push > Cannot tar (%+v) : %v", c.Files, errTar.Error()),
tmpDirectory, err := workerruntime.TmpDirectory(wk.currentJob.context)
if err != nil {
err = sdk.Error{
Message: "worker cache push > Cannot get tmp directory : " + err.Error(),
Status: http.StatusInternalServerError,
}
log.Error(ctx, "%v", err)
writeError(w, r, err)
return
}

tarF, err := afero.TempFile(wk.BaseDir(), tmpDirectory.Name(), "tar-")
if err != nil {
err = sdk.Error{
Message: "worker cache push > Cannot create tmp tar file : " + err.Error(),
Status: http.StatusInternalServerError,
}
log.Error(ctx, "%v", err)
writeError(w, r, err)
return
}
defer tarF.Close() // nolint

if err := sdk.CreateTarFromPaths(afero.NewOsFs(), c.WorkingDirectory, c.Files, tarF, nil); err != nil {
err = sdk.Error{
Message: fmt.Sprintf("worker cache push > Cannot tar (%+v) : %v", c.Files, err.Error()),
Status: http.StatusBadRequest,
}
log.Error(ctx, "%v", errTar)
writeError(w, r, errTar)
log.Error(ctx, "%v", err)
writeError(w, r, err)
return
}

tarInfo, err := tarF.Stat()
if err != nil {
err = sdk.Error{
Message: "worker cache push > Cannot get tmp tar file info : " + err.Error(),
Status: http.StatusInternalServerError,
}
log.Error(ctx, "%v", err)
writeError(w, r, err)
return
}

params := wk.currentJob.wJob.Parameters
projectKey := sdk.ParameterValue(params, "cds.project")
if projectKey == "" {
errP := sdk.Error{
err := sdk.Error{
Message: "worker cache push > Cannot find project",
Status: http.StatusInternalServerError,
}
log.Error(ctx, "%v", errP)
writeError(w, r, errP)
log.Error(ctx, "%v", err)
writeError(w, r, err)
return
}

var errPush error
for i := 0; i < 10; i++ {
if errPush = wk.client.WorkflowCachePush(projectKey, sdk.DefaultIfEmptyStorage(c.IntegrationName), vars["ref"], res, size); errPush == nil {
return
// Seek to be able to read the content of the file from beginning just after it had been written or in case of retry
if _, err := tarF.Seek(0, 0); err != nil {
errPush = err
} else {
if errPush = wk.client.WorkflowCachePush(projectKey, sdk.DefaultIfEmptyStorage(c.IntegrationName), c.Tag, tarF, int(tarInfo.Size())); errPush == nil {
return
}
}
time.Sleep(3 * time.Second)
log.Error(ctx, "worker cache push > cannot push cache (retry x%d) : %v", i, errPush)
}

err := sdk.Error{
err = sdk.Error{
Message: "worker cache push > Cannot push cache: " + errPush.Error(),
Status: http.StatusInternalServerError,
}
Expand All @@ -86,23 +124,25 @@ func cachePushHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc {
}

func cachePullHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
path := r.FormValue("path")
integrationName := sdk.DefaultIfEmptyStorage(r.FormValue("integration"))
return func(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
path := req.FormValue("path")
integrationName := sdk.DefaultIfEmptyStorage(req.FormValue("integration"))
params := wk.currentJob.wJob.Parameters
projectKey := sdk.ParameterValue(params, "cds.project")
bts, err := wk.client.WorkflowCachePull(projectKey, integrationName, vars["ref"])
r, err := wk.client.WorkflowCachePull(projectKey, integrationName, vars["ref"])
if err != nil {
err = sdk.Error{
Message: "worker cache pull > Cannot pull cache: " + err.Error(),
Status: http.StatusNotFound,
}
writeError(w, r, err)
writeError(w, req, err)
return
}

tr := tar.NewReader(bts)
log.Debug("cachePullHandler> Start read cache tar")

tr := tar.NewReader(r)
for {
header, errH := tr.Next()
if errH == io.EOF {
Expand All @@ -122,6 +162,8 @@ func cachePullHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc {
continue
}

log.Debug("cachePullHandler> Tar contains file %s", header.Name)

// the target location where the dir/file should be created
target := filepath.Join(path, header.Name)

Expand Down Expand Up @@ -163,6 +205,8 @@ func cachePullHandler(ctx context.Context, wk *CurrentWorker) http.HandlerFunc {
}
}

log.Debug("cachePullHandler> Create file at %s", target)

f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, os.FileMode(header.Mode))
if err != nil {
sdkErr := sdk.Error{
Expand Down
Loading

0 comments on commit 724564e

Please sign in to comment.