feat(api, worker): openstack integration storage (#3900)

* feat(api, worker): openstack integration storage

artifactUpload, artifactDownlaod, ServeStaticFiles Actions - and worker cache command can now use a dedicated openstack tenant

Signed-off-by: Yvonnick Esnault <[email protected]>

.gitignore

@@ -21,6 +21,9 @@ cover.html
 .bash_aliases
 npm-debug.log
 debug.test
+myOpenstackIntegration.yml
+test_results.yml
+*.dump
 .tmp
 contrib/**/dist
 

contrib/README.md

@@ -6,25 +6,30 @@ CDS support several kind of extensions:
 
 - Actions
 - Plugins
-- Secret Backends
+- worker-models
+- workflow-templates
 - µServices
 
 See [CDS documentation](https://github.com/ovh/cds) for more details.
 
 ## Actions
 
-See [User Actions Documentation](https://ovh.github.io/cds/workflows/pipelines/pipelines.actions/)
+See [User Actions Documentation](https://ovh.github.io/cds/workflows/pipelines/actions/user/)
 
 ## Plugins
 
-See [Plugins Documentation](https://ovh.github.io/cds/workflows/pipelines/pipelines.actions.user-actions/)
+See [Plugins Documentation](https://ovh.github.io/cds/workflows/pipelines/actions/plugins/)
+
+## Worker Models
+
+See [Worker Models Documentation](https://ovh.github.io/cds/workflows/pipelines/requirements/worker-model/)
+
+## Workflow Templates
+
+See [Workflow Templates Documentation](https://ovh.github.io/cds/workflows/template/)
 
 ## µServices
 
 - [cds2es](https://github.com/ovh/cds/tree/master/contrib/uservices/cds2es)
 - [cds2http](https://github.com/ovh/cds/tree/master/contrib/uservices/cds2http)
 - [Hubot XMPP](https://github.com/ovh/cds/tree/master/contrib/uservices/hubot-xmpp)
-
-## Contributions
-
-By convention, plugins must have prefix `plugin`.

contrib/integrations/arsenal/plugin-arsenal/Makefile

@@ -7,7 +7,7 @@ BUILDTIME := `date "+%m/%d/%y-%H:%M:%S"`
 TARGET_DIR = ./dist
 TARGET_NAME = arsenal
 
-PLUGIN_MANIFEST=`echo $TARGET_NAME.yml`
+PLUGIN_MANIFEST=`echo plugin.yml`
 export PLUGIN_MANIFEST
 
 define PLUGIN_MANIFEST_BINARY
@@ -33,7 +33,7 @@ clean:
 	@rm -rf $(TARGET_DIR)
 
 build: $(TARGET_DIR)
-	@cp $(TARGET_NAME).yml $(TARGET_DIR)/plugin.yml
+	@cp plugin.yml $(TARGET_DIR)/plugin.yml
 	@for GOOS in $(TARGET_OS); do \
 		for GOARCH in $(TARGET_ARCH); do \
 			EXTENSION=""; \

contrib/integrations/arsenal/plugin-arsenal/arsenal.yml → contrib/integrations/arsenal/plugin-arsenal/plugin.yml

@@ -1,5 +1,5 @@
 name: arsenal-deployment-plugin
 type: integration-deploy_application
 integration: Arsenal
-author: "François Samin"
+author: "OVH SAS"
 description: "OVH Arsenal Deployment Plugin"

contrib/integrations/hello/plugin-hello/plugin.yml → contrib/integrations/hello/plugin-hello-deployment/plugin.yml

@@ -1,5 +1,5 @@
 name: hello-deployment-plugin
-type: integration
+type: integration-deploy_application
 integration: Hello
-author: "Yvonnick Esnault"
+author: "OVH SAS"
 description: "Hello Example Deployment Plugin"

contrib/integrations/kubernetes/plugin-kubernetes/Makefile

@@ -7,7 +7,7 @@ BUILDTIME := `date "+%m/%d/%y-%H:%M:%S"`
 TARGET_DIR = ./dist
 TARGET_NAME = kubernetes
 
-PLUGIN_MANIFEST=`echo $TARGET_NAME.yml`
+PLUGIN_MANIFEST=`echo plugin.yml`
 export PLUGIN_MANIFEST
 
 define PLUGIN_MANIFEST_BINARY
@@ -37,7 +37,7 @@ clean:
 	@rm -rf $(TARGET_DIR)
 
 build: $(TARGET_DIR)
-	@cp $(TARGET_NAME).yml $(TARGET_DIR)/plugin.yml
+	@cp plugin.yml $(TARGET_DIR)/plugin.yml
 	@for GOOS in $(TARGET_OS); do \
 		for GOARCH in $(TARGET_ARCH); do \
 			EXTENSION=""; \
@@ -60,6 +60,6 @@ publish:
 			EXTENSION=""; \
 			if test "$$GOOS" = "windows" ; then EXTENSION=".exe"; fi; \
 			echo "Updating plugin binary $(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION"; \
-			cdsctl admin plugins binary-add kubernetes-deployment-plugin $(TARGET_DIR)/plugin-$$GOOS-$$GOARCH.yml $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \
+			cdsctl admin plugins binary-add plugin-kubernetes-deployment $(TARGET_DIR)/plugin-$$GOOS-$$GOARCH.yml $(TARGET_DIR)/$(TARGET_NAME)-$$GOOS-$$GOARCH$$EXTENSION; \
 		done; \
 	done

contrib/integrations/kubernetes/plugin-kubernetes/kubernetes.yml

@@ -1,5 +1,5 @@
-name: kubernetes-deployment-plugin
-type: integration
+name: plugin-kubernetes-deployment
+type: integration-deploy_application
 integration: Kubernetes
-author: "Benjamin Coenen"
+author: "OVH SAS"
 description: "Kubernetes Deployment Plugin"

contrib/integrations/kubernetes/plugin-kubernetes/main.go

@@ -33,19 +33,19 @@ import (
 This plugin have to be used as a deployment integration plugin
 
 Kubernetes deployment plugin must configured as following:
-	name: kubernetes-deployment-plugin
+	name: plugin-kubernetes-deployment
 	type: integration
 	author: "Benjamin Coenen"
 	description: "Kubernetes Deployment Plugin"
 
-$ cdsctl admin plugins import kubernetes-deployment-plugin.yml
+$ cdsctl admin plugins import plugin-kubernetes-deployment.yml
 
 Build the present binaries and import in CDS:
 	os: linux
 	arch: amd64
 	cmd: <path-to-binary-file>
 
-$ cdsctl admin plugins binary-add kubernetes-deployment-plugin kubernetes-deployment-plugin-bin.yml <path-to-binary-file>
+$ cdsctl admin plugins binary-add plugin-kubernetes-deployment plugin-kubernetes-deployment-bin.yml <path-to-binary-file>
 */
 const (
 	kubectlLink = "https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/"

contrib/integrations/openstack/README.md

@@ -0,0 +1,17 @@
+# How to import a OpenStack configuration
+
+An OpenStack Integration use the builtin integration model named **Openstack**.
+
+You can set an OpenStack configuration on a CDS Project.
+
+If you want to set a global OpenStack configuration, available on all CDS Projects, you 
+have just to set the attribute **public** to `true` in the openstack.yml file.
+
+```bash
+
+# 1 - edit the openstack.yml file
+
+# 2 - run 
+$ cdsctl project integration import openstack.yml
+
+```

contrib/integrations/openstack/openstack.yml

@@ -0,0 +1,29 @@
+name: MyOpenstackTenant
+model:
+  name: Openstack
+  public: false
+config:
+  address:
+    value: https://auth.cloud.ovh.net/v2.0/
+    type: string
+  domain:
+    value: ""
+    type: string
+  password:
+    value: 'your-password-here'
+    type: password
+  region:
+    value: your-region
+    type: string
+  storage_container_prefix:
+    value: cds-prefix-
+    type: string
+  storage_temporary_url_supported:
+    value: "true"
+    type: string
+  tenant_name:
+    value: "your-openstack-tenant"
+    type: string
+  username:
+    value: your-openstack-user
+    type: string

engine/api/action/builtin.go

@@ -284,6 +284,13 @@ Semver used if fully compatible with https://semver.org/
 		Type:        sdk.StringParameter,
 		Value:       "",
 		Advanced:    true})
+	serveStaticAct.Parameter(sdk.Parameter{
+		Name:        "destination",
+		Description: "Destination of uploading. Use the name of integration attached on your project",
+		Value:       "", // empty is the default value
+		Type:        sdk.StringParameter,
+		Advanced:    true,
+	})
 
 	return checkBuiltinAction(db, serveStaticAct)
 }
@@ -328,17 +335,15 @@ func CreateBuiltinArtifactActions(db *gorp.DbMap) error {
 	// Check ArtifactUpload action
 	err := db.QueryRow(query, sdk.ArtifactUpload, sdk.BuiltinAction).Scan(&name)
 	if err != nil && err == sql.ErrNoRows {
-		err = createBuiltinArtifactUploadAction(db)
-		if err != nil {
+		if err := createBuiltinArtifactUploadAction(db); err != nil {
 			return sdk.WrapError(err, "cannot create builtin artifact upload action")
 		}
 	}
 
 	// Check ArtifactDownload action
 	err = db.QueryRow(query, sdk.ArtifactDownload, sdk.BuiltinAction).Scan(&name)
 	if err != nil && err == sql.ErrNoRows {
-		err = createBuiltinArtifactDownloadAction(db)
-		if err != nil {
+		if err := createBuiltinArtifactDownloadAction(db); err != nil {
 			return sdk.WrapError(err, "cannot create builtin artifact download action")
 		}
 	}
@@ -363,6 +368,13 @@ func createBuiltinArtifactUploadAction(db *gorp.DbMap) error {
 		Type:        sdk.BooleanParameter,
 		Description: "Enable artifact upload",
 		Value:       "true"})
+	upload.Parameter(sdk.Parameter{
+		Name:        "destination",
+		Description: "Destination of this artifact. Use the name of integration attached on your project",
+		Value:       "", // empty is the default value
+		Type:        sdk.StringParameter,
+		Advanced:    true,
+	})
 
 	tx, err := db.Begin()
 	if err != nil {

engine/api/api.go

@@ -227,6 +227,7 @@ type API struct {
 	Router              *Router
 	Config              Configuration
 	DBConnectionFactory *database.DBConnectionFactory
+	SharedStorage       objectstore.Driver
 	StartupTime         time.Time
 	Maintenance         bool
 	eventsBroker        *eventsBroker
@@ -535,8 +536,12 @@ func (a *API) Serve(ctx context.Context) error {
 		},
 	}
 
-	if err := objectstore.Initialize(ctx, cfg); err != nil {
-		return fmt.Errorf("cannot initialize storage: %v", err)
+	// DEPRECATED
+	// API Storage will be a public integration
+	var errStorage error
+	a.SharedStorage, errStorage = objectstore.Init(ctx, cfg)
+	if errStorage != nil {
+		return fmt.Errorf("cannot initialize storage: %v", errStorage)
 	}
 
 	log.Info("Initializing database connection...")

engine/api/api_routes.go

@@ -220,6 +220,7 @@ func (api *API) InitRouter() {
 
 	// Workflows run
 	r.Handle("/project/{permProjectKey}/runs", r.GET(api.getWorkflowAllRunsHandler, EnableTracing()))
+	r.Handle("/project/{key}/workflows/{permWorkflowName}/artifact/{artifactId}", r.GET(api.getDownloadArtifactHandler))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs", r.GET(api.getWorkflowRunsHandler, EnableTracing()), r.POSTEXECUTE(api.postWorkflowRunHandler, AllowServices(true), EnableTracing()))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/latest", r.GET(api.getLatestWorkflowRunHandler))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/tags", r.GET(api.getWorkflowRunTagsHandler))
@@ -236,7 +237,6 @@ func (api *API) InitRouter() {
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/{number}/nodes/{nodeRunID}/job/{runJobId}/info", r.GET(api.getWorkflowNodeRunJobSpawnInfosHandler))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/{number}/nodes/{nodeRunID}/job/{runJobId}/log/service", r.GET(api.getWorkflowNodeRunJobServiceLogsHandler))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/{number}/nodes/{nodeRunID}/job/{runJobId}/step/{stepOrder}", r.GET(api.getWorkflowNodeRunJobStepHandler))
-	r.Handle("/project/{key}/workflows/{permWorkflowName}/artifact/{artifactId}", r.GET(api.getDownloadArtifactHandler))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/node/{nodeID}/triggers/condition", r.GET(api.getWorkflowTriggerConditionHandler))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/{number}/nodes/{nodeRunID}/release", r.POST(api.releaseApplicationWorkflowHandler))
 	r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/{number}/hooks/{hookRunID}/callback", r.POST(api.postWorkflowJobHookCallbackHandler, AllowServices(true)))
@@ -260,13 +260,16 @@ func (api *API) InitRouter() {
 	// Export Environment
 	r.Handle("/project/{permProjectKey}/export/environment/{environmentName}", r.GET(api.getEnvironmentExportHandler))
 
-	// Artifacts
-	r.Handle("/staticfiles/store", r.GET(api.getStaticFilesStoreHandler, Auth(false)))
-	r.Handle("/artifact/store", r.GET(api.getArtifactsStoreHandler, Auth(false)))
+	// Project storage
+	r.Handle("/project/{permProjectKey}/storage/{integrationName}", r.GET(api.getArtifactsStoreHandler, Auth(false)))
+	r.Handle("/project/{permProjectKey}/storage/{integrationName}/artifact/{ref}", r.POSTEXECUTE(api.postWorkflowJobArtifactHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
+	r.Handle("/project/{permProjectKey}/storage/{integrationName}/artifact/{ref}/url", r.POSTEXECUTE(api.postWorkflowJobArtifacWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
+	r.Handle("/project/{permProjectKey}/storage/{integrationName}/artifact/{ref}/url/callback", r.POSTEXECUTE(api.postWorkflowJobArtifactWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
+	r.Handle("/project/{permProjectKey}/storage/{integrationName}/staticfiles/{name}", r.POSTEXECUTE(api.postWorkflowJobStaticFilesHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
 
 	// Cache
-	r.Handle("/project/{permProjectKey}/cache/{tag}", r.POSTEXECUTE(api.postPushCacheHandler, NeedWorker()), r.GET(api.getPullCacheHandler, NeedWorker()))
-	r.Handle("/project/{permProjectKey}/cache/{tag}/url", r.POSTEXECUTE(api.postPushCacheWithTempURLHandler, NeedWorker()), r.GET(api.getPullCacheWithTempURLHandler, NeedWorker()))
+	r.Handle("/project/{permProjectKey}/storage/{integrationName}/cache/{tag}", r.POSTEXECUTE(api.postPushCacheHandler, NeedWorker()), r.GET(api.getPullCacheHandler, NeedWorker()))
+	r.Handle("/project/{permProjectKey}/storage/{integrationName}/cache/{tag}/url", r.POSTEXECUTE(api.postPushCacheWithTempURLHandler, NeedWorker()), r.GET(api.getPullCacheWithTempURLHandler, NeedWorker()))
 
 	//Workflow queue
 	r.Handle("/queue/workflows", r.GET(api.getWorkflowJobQueueHandler, EnableTracing(), MaintenanceAware()))
@@ -285,12 +288,6 @@ func (api *API) InitRouter() {
 	r.Handle("/queue/workflows/{permID}/tag", r.POSTEXECUTE(api.postWorkflowJobTagsHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
 	r.Handle("/queue/workflows/{permID}/variable", r.POSTEXECUTE(api.postWorkflowJobVariableHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
 	r.Handle("/queue/workflows/{permID}/step", r.POSTEXECUTE(api.postWorkflowJobStepStatusHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
-	r.Handle("/queue/workflows/{permID}/artifact/{ref}", r.POSTEXECUTE(api.postWorkflowJobArtifactHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
-	r.Handle("/queue/workflows/{permID}/artifact/{ref}/url", r.POSTEXECUTE(api.postWorkflowJobArtifacWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
-	r.Handle("/queue/workflows/{permID}/artifact/{ref}/url/callback", r.POSTEXECUTE(api.postWorkflowJobArtifactWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
-	r.Handle("/queue/workflows/{permID}/staticfiles/{name}", r.POSTEXECUTE(api.postWorkflowJobStaticFilesHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
-	r.Handle("/queue/workflows/{permID}/staticfiles/{name}/url", r.POSTEXECUTE(api.postWorkflowJobStaticFilesWithTempURLHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
-	r.Handle("/queue/workflows/{permID}/staticfiles/{name}/url/callback", r.POSTEXECUTE(api.postWorkflowJobStaticFilesWithTempURLCallbackHandler, NeedWorker(), EnableTracing(), MaintenanceAware()))
 
 	r.Handle("/variable/type", r.GET(api.getVariableTypeHandler))
 	r.Handle("/parameter/type", r.GET(api.getParameterTypeHandler))

engine/api/application_deployment_test.go

@@ -55,7 +55,7 @@ func Test_postApplicationDeploymentStrategyConfigHandler(t *testing.T) {
 	test.NoError(t, application.Insert(api.mustDB(), api.Cache, proj, app, u))
 
 	pf := sdk.IntegrationModel{
-		Name:       "test-deploy-2",
+		Name:       "test-deploy-post-2" + pkey,
 		Deployment: true,
 	}
 	test.NoError(t, integration.InsertModel(db, &pf))
@@ -178,7 +178,7 @@ func Test_postApplicationDeploymentStrategyConfigHandler_InsertTwoDifferentInteg
 	test.NoError(t, application.Insert(api.mustDB(), api.Cache, proj, app, u))
 
 	pf := sdk.IntegrationModel{
-		Name:       "test-deploy-2",
+		Name:       "test-deploy-TwoDifferentIntegrations-2" + pkey,
 		Deployment: true,
 	}
 	test.NoError(t, integration.InsertModel(db, &pf))
@@ -307,7 +307,7 @@ func Test_postApplicationDeploymentStrategyConfigHandlerAsProvider(t *testing.T)
 	test.NoError(t, application.Insert(api.mustDB(), api.Cache, proj, app, u))
 
 	pf := sdk.IntegrationModel{
-		Name:       "test-deploy-3",
+		Name:       "test-deploy-3" + pkey,
 		Deployment: true,
 		DeploymentDefaultConfig: sdk.IntegrationConfig{
 			"token": sdk.IntegrationConfigValue{