feat(api,ui,worker): add serve static files (close #3567) (#3618)

docs/content/workflows/pipelines/actions/builtin/serve-static-files.md

@@ -0,0 +1,29 @@
++++
+title = "Serve Static Files"
+chapter = true
++++
+
+**Serve Static Files Action** is a builtin action, you can't modify it.
+
+This action can be used to upload static files and serve them. For example your HTML report about coverage, tests, performances, ...
+
+Pay attention this action is only available if your objectstore is configured to use Openstack Swift. And fow now by default your static files will be deleted after 2 months.
+
+## Parameters
+* name: Name to display in CDS UI and identify your static files
+* path: Path where static files will be uploaded (example: mywebsite/*). If it's a file, the entrypoint would be set to this filename by default.
+* tag: Filename (and not path) for the entrypoint when serving static files (default: if empty it would be index.html).
+
+### Example
+
+* In this example I created a website with script in bash and use action `Serve Static Files`.
+
+![img](/images/workflows.pipelines.actions.builtin.serve-static-files-job.png)
+
+* Launch pipeline, check logs
+
+![img](/images/workflows.pipelines.actions.builtin.serve-static-files-logs.png)
+
+* View your static files with links in the tab artifact
+
+![img](/images/workflows.pipelines.actions.builtin.serve-static-files-tab.png)

engine/api/action/builtin.go

@@ -248,7 +248,32 @@ Semver used if fully compatible with https://semver.org/
 		Description: "Set a list of artifacts, separate by , . You can also use regexp.",
 		Type:        sdk.StringParameter,
 	})
-	return checkBuiltinAction(db, gitrelease)
+	if err := checkBuiltinAction(db, gitrelease); err != nil {
+		return err
+	}
+
+	// ----------------------------------- Serve Static Files -----------------------
+	serveStaticAct := sdk.NewAction(sdk.ServeStaticFiles)
+	serveStaticAct.Type = sdk.BuiltinAction
+	serveStaticAct.Description = `CDS Builtin Action
+	Useful to upload static files and serve them.
+	For example your report about coverage, tests, performances, ...`
+	serveStaticAct.Parameter(sdk.Parameter{
+		Name:        "name",
+		Description: "Name to display in CDS UI and identify your static files",
+		Type:        sdk.StringParameter})
+	serveStaticAct.Parameter(sdk.Parameter{
+		Name:        "path",
+		Description: "Path where static files will be uploaded (example: mywebsite/*). If it's a file, the entrypoint would be set to this filename by default.",
+		Type:        sdk.StringParameter})
+	serveStaticAct.Parameter(sdk.Parameter{
+		Name:        "entrypoint",
+		Description: "Filename (and not path) for the entrypoint when serving static files (default: if empty it would be index.html)",
+		Type:        sdk.StringParameter,
+		Value:       "",
+		Advanced:    true})
+
+	return checkBuiltinAction(db, serveStaticAct)
 }
 
 // checkBuiltinAction add builtin actions in database if needed

engine/api/api_routes.go

@@ -303,6 +303,7 @@ func (api *API) InitRouter() {
 	r.Handle("/project/{key}/application/{permApplicationName}/pipeline/{permPipelineKey}/{buildNumber}/artifact/{tag}/url/callback", r.POSTEXECUTE(api.postArtifactWithTempURLCallbackHandler))
 	r.Handle("/project/{key}/application/{permApplicationName}/pipeline/{permPipelineKey}/{buildNumber}/artifact/{tag}", r.POSTEXECUTE(api.uploadArtifactHandler))
 	r.Handle("/project/{key}/application/{permApplicationName}/pipeline/{permPipelineKey}/artifact/download/{id}", r.GET(api.downloadArtifactHandler))
+	r.Handle("/staticfiles/store", r.GET(api.getStaticFilesStoreHandler, Auth(false)))
 	r.Handle("/artifact/store", r.GET(api.getArtifactsStoreHandler, Auth(false)))
 	r.Handle("/artifact/{hash}", r.GET(api.downloadArtifactDirectHandler, Auth(false)))
 
@@ -345,6 +346,9 @@ func (api *API) InitRouter() {
 	r.Handle("/queue/workflows/{permID}/artifact/{ref}", r.POSTEXECUTE(api.postWorkflowJobArtifactHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
 	r.Handle("/queue/workflows/{permID}/artifact/{ref}/url", r.POSTEXECUTE(api.postWorkflowJobArtifacWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
 	r.Handle("/queue/workflows/{permID}/artifact/{ref}/url/callback", r.POSTEXECUTE(api.postWorkflowJobArtifactWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
+	r.Handle("/queue/workflows/{permID}/staticfiles/{name}", r.POSTEXECUTE(api.postWorkflowJobStaticFilesHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
+	r.Handle("/queue/workflows/{permID}/staticfiles/{name}/url", r.POSTEXECUTE(api.postWorkflowJobStaticFilesWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
+	r.Handle("/queue/workflows/{permID}/staticfiles/{name}/url/callback", r.POSTEXECUTE(api.postWorkflowJobStaticFilesWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
 
 	r.Handle("/variable/type", r.GET(api.getVariableTypeHandler))
 	r.Handle("/parameter/type", r.GET(api.getParameterTypeHandler))

engine/api/objectstore/fs.go

@@ -34,6 +34,11 @@ func (fss *FilesystemStore) Status() sdk.MonitoringStatusLine {
 	return sdk.MonitoringStatusLine{Component: "Object-Store", Value: "Filesystem Storage", Status: sdk.MonitoringStatusOK}
 }
 
+// ServeStaticFiles NOT YET IMPLEMENTED
+func (fss *FilesystemStore) ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) {
+	return "", sdk.ErrNotImplemented
+}
+
 // Store store a object on disk
 func (fss *FilesystemStore) Store(o Object, data io.ReadCloser) (string, error) {
 	dst := path.Join(fss.basedir, o.GetPath())

engine/api/objectstore/objectstore.go

@@ -39,6 +39,14 @@ func Store(o Object, data io.ReadCloser) (string, error) {
 	return "", fmt.Errorf("store not initialized")
 }
 
+//ServeStaticFiles send files to serve static files with the entrypoint of html page and return public URL taking a tar file
+func ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) {
+	if storage != nil {
+		return storage.ServeStaticFiles(o, entrypoint, data)
+	}
+	return "", fmt.Errorf("store not initialized")
+}
+
 //Fetch an object with default objectstore driver
 func Fetch(o Object) (io.ReadCloser, error) {
 	if storage != nil {
@@ -76,6 +84,7 @@ func FetchTempURL(o Object) (string, error) {
 type Driver interface {
 	Status() sdk.MonitoringStatusLine
 	Store(o Object, data io.ReadCloser) (string, error)
+	ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error)
 	Fetch(o Object) (io.ReadCloser, error)
 	Delete(o Object) error
 }
@@ -86,6 +95,10 @@ type DriverWithRedirect interface {
 	StoreURL(o Object) (url string, key string, err error)
 	// FetchURL returns a temporary url and a secret key to fetch an object
 	FetchURL(o Object) (url string, key string, err error)
+	// ServeStaticFilesURL returns a temporary url and a secret key to serve static files in a container
+	ServeStaticFilesURL(o Object, entrypoint string) (string, string, error)
+	// GetPublicURL returns a public url to fetch an object (check your object ACLs before)
+	GetPublicURL(o Object) (url string, err error)
 }
 
 // Initialize setup wanted ObjectStore driver

engine/api/objectstore/ssh.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 
 	"github.com/ovh/cds/sdk"
+
 	"golang.org/x/crypto/ssh"
 )
 

engine/api/objectstore/swift.go

@@ -49,13 +49,48 @@ func (s *SwiftStore) Status() sdk.MonitoringStatusLine {
 	}
 }
 
+// ServeStaticFiles send files to serve static files with the entrypoint of html page and return public URL taking a tar file
+func (s *SwiftStore) ServeStaticFiles(o Object, entrypoint string, data io.ReadCloser) (string, error) {
+	container := s.containerprefix + o.GetPath()
+	object := o.GetName()
+	escape(container, object)
+	log.Debug("SwiftStore> Storing /%s/%s\n", container, object)
+
+	if entrypoint == "" {
+		entrypoint = "index.html"
+	}
+	headers := map[string]string{
+		"X-Web-Mode":                    "TRUE",
+		"X-Container-Meta-Web-Listings": "TRUE",
+		"X-Container-Meta-Web-Index":    entrypoint,
+		"X-Container-Read":              ".r:*,.rlistings",
+		"X-Delete-After":                fmt.Sprintf("%d", time.Now().Add(time.Hour*1500).Unix()), //TODO: to delete when purge will be developed
+	}
+
+	log.Debug("SwiftStore> creating container %s", container)
+	if err := s.ContainerCreate(container, headers); err != nil {
+		return "", sdk.WrapError(err, "Unable to create container %s", container)
+	}
+
+	log.Debug("SwiftStore> creating object %s/%s", container, object)
+	res, errU := s.BulkUpload(container, data, "tar", nil)
+	if errU != nil {
+		return "", sdk.WrapError(errU, "SwiftStore> Unable to bulk upload %s : %v : %+v", object, errU, res.Errors)
+	}
+
+	if err := data.Close(); err != nil {
+		return "", sdk.WrapError(err, "Unable to close data buffer")
+	}
+
+	return s.StorageUrl + "/" + container, nil
+}
+
 // Store stores in swift
 func (s *SwiftStore) Store(o Object, data io.ReadCloser) (string, error) {
 	container := s.containerprefix + o.GetPath()
 	object := o.GetName()
 	escape(container, object)
 	log.Debug("SwiftStore> Storing /%s/%s\n", container, object)
-
 	log.Debug("SwiftStore> creating container %s", container)
 	if err := s.ContainerCreate(container, nil); err != nil {
 		return "", sdk.WrapError(err, "Unable to create container %s", container)
@@ -129,7 +164,6 @@ func (s *SwiftStore) StoreURL(o Object) (string, string, error) {
 	container := s.containerprefix + o.GetPath()
 	object := o.GetName()
 	escape(container, object)
-
 	if err := s.ContainerCreate(container, nil); err != nil {
 		return "", "", sdk.WrapError(err, "Unable to create container %s", container)
 	}
@@ -143,6 +177,39 @@ func (s *SwiftStore) StoreURL(o Object) (string, string, error) {
 	return url, string(key), nil
 }
 
+// GetPublicURL returns a public url to fetch an object (check your object ACLs before)
+func (s *SwiftStore) GetPublicURL(o Object) (url string, err error) {
+	return s.StorageUrl + "/" + (s.containerprefix + o.GetPath()), nil
+}
+
+// ServeStaticFilesURL returns a temporary url and a secret key to serve static files in a container
+func (s *SwiftStore) ServeStaticFilesURL(o Object, entrypoint string) (string, string, error) {
+	container := s.containerprefix + o.GetPath()
+	object := o.GetName()
+	escape(container, object)
+	if entrypoint == "" {
+		entrypoint = "index.html"
+	}
+
+	headers := map[string]string{
+		"X-Web-Mode":                    "TRUE",
+		"X-Container-Meta-Web-Listings": "TRUE",
+		"X-Container-Meta-Web-Index":    entrypoint,
+		"X-Container-Read":              ".r:*,.rlistings",
+	}
+	if err := s.ContainerCreate(container, headers); err != nil {
+		return "", "", sdk.WrapError(err, "Unable to create container %s", container)
+	}
+
+	key, err := s.containerKey(container)
+	if err != nil {
+		return "", "", sdk.WrapError(err, "Unable to get container key %s", container)
+	}
+
+	url := s.ObjectTempUrl(container, object, string(key), "PUT", time.Now().Add(time.Hour))
+	return url, string(key), nil
+}
+
 func (s *SwiftStore) containerKey(container string) (string, error) {
 	_, headers, err := s.Container(container)
 	if err != nil {
@@ -178,5 +245,5 @@ func (s *SwiftStore) FetchURL(o Object) (string, string, error) {
 	url := s.ObjectTempUrl(container, object, string(key), "GET", time.Now().Add(time.Hour))
 
 	log.Debug("SwiftStore> Fetch URL: %s", string(url))
-	return url, string(key), nil
+	return url + "&extract-archive=tar.gz", string(key), nil
 }

engine/api/workflow/dao_node_run.go

@@ -93,6 +93,13 @@ func LoadNodeRun(db gorp.SqlExecutor, projectkey, workflowname string, number, i
 		}
 		r.Artifacts = arts
 	}
+	if loadOpts.WithStaticFiles {
+		staticFiles, errS := loadStaticFilesByNodeRunID(db, r.ID)
+		if errS != nil {
+			return nil, sdk.WrapError(errS, "LoadNodeRun>Error loading static files for run %d", r.ID)
+		}
+		r.StaticFiles = staticFiles
+	}
 	if loadOpts.WithCoverage {
 		cov, errCov := LoadCoverageReport(db, r.ID)
 		if errCov != nil && !sdk.ErrorIs(errCov, sdk.ErrNotFound) {
@@ -146,6 +153,14 @@ func LoadNodeRunByNodeJobID(db gorp.SqlExecutor, nodeJobRunID int64, loadOpts Lo
 		r.Artifacts = arts
 	}
 
+	if loadOpts.WithStaticFiles {
+		staticFiles, errS := loadStaticFilesByNodeRunID(db, r.ID)
+		if errS != nil {
+			return nil, sdk.WrapError(errS, "LoadNodeRunByNodeJobID>Error loading static files for run %d", r.ID)
+		}
+		r.StaticFiles = staticFiles
+	}
+
 	return r, nil
 
 }
@@ -198,11 +213,19 @@ func LoadNodeRunByID(db gorp.SqlExecutor, id int64, loadOpts LoadRunOptions) (*s
 	if loadOpts.WithArtifacts {
 		arts, errA := loadArtifactByNodeRunID(db, r.ID)
 		if errA != nil {
-			return nil, sdk.WrapError(errA, "LoadNodeRun>Error loading artifacts for run %d", r.ID)
+			return nil, sdk.WrapError(errA, "LoadNodeRunByID>Error loading artifacts for run %d", r.ID)
 		}
 		r.Artifacts = arts
 	}
 
+	if loadOpts.WithStaticFiles {
+		staticFiles, errS := loadStaticFilesByNodeRunID(db, r.ID)
+		if errS != nil {
+			return nil, sdk.WrapError(errS, "LoadNodeRunByID>Error loading static files for run %d", r.ID)
+		}
+		r.StaticFiles = staticFiles
+	}
+
 	return r, nil
 
 }

engine/api/workflow/dao_run.go

@@ -35,6 +35,7 @@ workflow_run.version
 type LoadRunOptions struct {
 	WithCoverage            bool
 	WithArtifacts           bool
+	WithStaticFiles         bool
 	WithTests               bool
 	WithLightTests          bool
 	WithVulnerabilities     bool
@@ -737,6 +738,14 @@ func syncNodeRuns(db gorp.SqlExecutor, wr *sdk.WorkflowRun, loadOpts LoadRunOpti
 			wnr.Artifacts = arts
 		}
 
+		if loadOpts.WithStaticFiles {
+			staticFiles, errS := loadStaticFilesByNodeRunID(db, wnr.ID)
+			if errS != nil {
+				return sdk.WrapError(errS, "syncNodeRuns>Error loading static files for node run %d", wnr.ID)
+			}
+			wnr.StaticFiles = staticFiles
+		}
+
 		if loadOpts.WithCoverage {
 			cov, errCov := LoadCoverageReport(db, wnr.ID)
 			if errCov != nil && !sdk.ErrorIs(errCov, sdk.ErrNotFound) {

engine/api/workflow/dao_staticfiles.go

@@ -0,0 +1,44 @@
+package workflow
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/go-gorp/gorp"
+
+	"github.com/ovh/cds/sdk"
+)
+
+func loadStaticFilesByNodeRunID(db gorp.SqlExecutor, nodeRunID int64) ([]sdk.StaticFiles, error) {
+	var dbstaticFiles []dbStaticFiles
+	if _, err := db.Select(&dbstaticFiles, `SELECT
+			id,
+			name,
+			entrypoint,
+			created,
+			public_url,
+			workflow_node_run_id
+		FROM workflow_node_run_static_files WHERE workflow_node_run_id = $1`, nodeRunID); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	staticFiles := make([]sdk.StaticFiles, len(dbstaticFiles))
+	for i := range dbstaticFiles {
+		staticFiles[i] = sdk.StaticFiles(dbstaticFiles[i])
+	}
+	return staticFiles, nil
+}
+
+// InsertStaticFiles insert in table workflow_artifacts
+func InsertStaticFiles(db gorp.SqlExecutor, sf *sdk.StaticFiles) error {
+	sf.Created = time.Now()
+	dbstaticFiles := dbStaticFiles(*sf)
+	if err := db.Insert(&dbstaticFiles); err != nil {
+		return err
+	}
+	sf.ID = dbstaticFiles.ID
+	return nil
+}