feat(api): support custom s3 API endpoint (eg. min.io) (#4673)

contrib/integrations/aws/README.md

@@ -1,17 +1,3 @@
 # How to import a Amazon AWS configuration
 
-An AWS Integration use the builtin integration model named **AWS**.
-
-You can set an AWS configuration on a CDS Project.
-
-If you want to set a global AWS configuration, available on all CDS Projects, you 
-have just to set the attribute **public** to `true` in the aws.yml file.
-
-```bash
-
-# 1 - edit the aws.yml file
-
-# 2 - run 
-$ cdsctl project integration import aws.yml
-
-```
+Read more at <https://ovh.github.io/cds/docs/integrations/aws/aws_s3/>

docs/content/docs/integrations/aws/aws_s3.md

@@ -32,7 +32,6 @@ Create a file project-configuration.yml:
 name: MyAWS
 model:
   name: AWS
-  public: false
 config:
   region:
     value: your-region
@@ -92,3 +91,53 @@ Import the integration with :
 ```bash
 cdsctl admin integration-model import public-configuration.yml
 ```
+
+### Using min.io as an alternative
+
+[Minio](https://min.io) is a Open Source, Enterprise-Grade, Amazon S3 Compatible Object Storage.
+
+According to https://docs.min.io/docs/how-to-use-aws-sdk-for-go-with-minio-server.html, you can define `endpoint`, `disable_ssl`, `force_path_style` to link CDS to a Minio server.
+
+For example, you can run a minio local server with the following docker command.
+
+```bash
+docker run -p 9000:9000 --name minio1 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -v /mnt/data:/data \
+  minio/minio server /data
+```
+
+Then you can import the following content to you project with the `cdsctl project integration import` command.
+
+```yaml
+name: local min.io
+model:
+  name: AWS
+storage: true
+config:
+  region:
+    value: us-east-1
+    type: string
+  bucket_name:
+    value: cds-storage
+    type: string
+  prefix:
+    value: cds-prefix-
+    type: string
+  access_key_id:
+    value: 'AKIAIOSFODNN7EXAMPLE'
+    type: string
+  secret_access_key:
+    value: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
+    type: password
+  endpoint:
+    value: 'http://localhost:9000'
+    type: string
+  disable_ssl:
+    value: 'true'
+    type: boolean
+  force_path_style:
+    value: 'true'
+    type: boolean
+```

engine/api/api.go

@@ -135,6 +135,9 @@ type Configuration struct {
 			AccessKeyID         string `toml:"accessKeyId" json:"accessKeyId" comment:"A static AWS Secret Key ID"`
 			SecretAccessKey     string `toml:"secretAccessKey" json:"-" comment:"A static AWS Secret Access Key"`
 			SessionToken        string `toml:"sessionToken" json:"-" comment:"A static AWS session token"`
+			Endpoint            string `toml:"endpoint" json:"endpoint" comment:"S3 API Endpoint (optional)" commented:"true"` //optional
+			DisableSSL          bool   `toml:"disableSSL" json:"disableSSL" commented:"true"`                                  //optional
+			ForcePathStyle      bool   `toml:"forcePathStyle" json:"forcePathStyle" commented:"true"`                          //optional
 		} `toml:"awss3" json:"awss3"`
 	} `toml:"artifact" comment:"Either filesystem local storage or Openstack Swift Storage are supported" json:"artifact"`
 	Features struct {
@@ -576,6 +579,9 @@ func (a *API) Serve(ctx context.Context) error {
 				BucketName:          a.Config.Artifact.AWSS3.BucketName,
 				Region:              a.Config.Artifact.AWSS3.Region,
 				SessionToken:        a.Config.Artifact.AWSS3.SessionToken,
+				Endpoint:            a.Config.Artifact.AWSS3.Endpoint,
+				DisableSSL:          a.Config.Artifact.AWSS3.DisableSSL,
+				ForcePathStyle:      a.Config.Artifact.AWSS3.ForcePathStyle,
 			},
 			Openstack: objectstore.ConfigOptionsOpenstack{
 				Address:         a.Config.Artifact.Openstack.URL,

engine/api/objectstore/awss3.go

@@ -38,6 +38,14 @@ func newS3Store(integration sdk.ProjectIntegration, conf ConfigOptionsAWSS3) (*A
 	} else {
 		aConf.Credentials = credentials.NewStaticCredentials(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken)
 	}
+
+	// If a custom endpoint is set, set up a new endPoint resolver (eg. minio)
+	if conf.Endpoint != "" {
+		aConf.Endpoint = aws.String(conf.Endpoint)
+		aConf.DisableSSL = aws.Bool(conf.DisableSSL)
+		aConf.S3ForcePathStyle = aws.Bool(conf.ForcePathStyle)
+	}
+
 	sess, err := session.NewSession(aConf)
 	if err != nil {
 		return nil, sdk.WrapError(err, "Unable to create an AWS session")

engine/api/objectstore/objectstore.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/go-gorp/gorp"
@@ -75,6 +76,9 @@ type ConfigOptionsAWSS3 struct {
 	AccessKeyID         string
 	SecretAccessKey     string
 	SessionToken        string
+	Endpoint            string //optional
+	DisableSSL          bool   //optional
+	ForcePathStyle      bool   //optional
 }
 
 // ConfigOptionsOpenstack is used by ConfigOptions
@@ -119,13 +123,19 @@ func initDriver(db gorp.SqlExecutor, projectKey, integrationName string) (Driver
 
 	switch projectIntegration.Model.Name {
 	case sdk.AWSIntegrationModel:
-		return newS3Store(projectIntegration, ConfigOptionsAWSS3{
+		cfg := ConfigOptionsAWSS3{
 			Region:          projectIntegration.Config["region"].Value,
 			BucketName:      projectIntegration.Config["bucket_name"].Value,
 			Prefix:          projectIntegration.Config["prefix"].Value,
 			AccessKeyID:     projectIntegration.Config["access_key_id"].Value,
 			SecretAccessKey: projectIntegration.Config["secret_access_key"].Value,
-		})
+		}
+		if endpoint := projectIntegration.Config["endpoint"].Value; endpoint != "" {
+			cfg.Endpoint = endpoint
+			cfg.DisableSSL, _ = strconv.ParseBool(projectIntegration.Config["disable_ssl"].Value)
+			cfg.ForcePathStyle, _ = strconv.ParseBool(projectIntegration.Config["force_path_style"].Value)
+		}
+		return newS3Store(projectIntegration, cfg)
 	case sdk.OpenstackIntegrationModel:
 		return newSwiftStore(projectIntegration, ConfigOptionsOpenstack{
 			Address:         projectIntegration.Config["address"].Value,

sdk/integration.go

@@ -120,6 +120,15 @@ var (
 			"secret_access_key": IntegrationConfigValue{
 				Type: IntegrationConfigTypePassword,
 			},
+			"endpoint": IntegrationConfigValue{
+				Type: IntegrationConfigTypeString,
+			},
+			"disable_ssl": IntegrationConfigValue{
+				Type: IntegrationConfigTypeBoolean,
+			},
+			"force_path_style": IntegrationConfigValue{
+				Type: IntegrationConfigTypeBoolean,
+			},
 		},
 		Storage:  true,
 		Disabled: false,
@@ -195,6 +204,8 @@ const (
 	IntegrationConfigTypeText = "text"
 	// IntegrationConfigTypePassword represents a password configuration value
 	IntegrationConfigTypePassword = "password"
+	// IntegrationConfigTypeBoolean represents a password configuration value
+	IntegrationConfigTypeBoolean = "boolean"
 )
 
 // IntegrationConfigValue represent a configuration value for a integration

ui/src/app/views/project/show/integrations/form/project.integration.form.html

@@ -49,14 +49,12 @@
                             </label>
                         </div>
                         <div class="ten wide field">
-                            <input type="text" [(ngModel)]="newIntegration.config[k].value"
-                                *ngIf="newIntegration.config[k].type === 'string'">
+                            <input type="checkbox" [(ngModel)]="newIntegration.config[k].value"  *ngIf="newIntegration.config[k].type === 'boolean'">
+                            <input type="text" [(ngModel)]="newIntegration.config[k].value"  *ngIf="newIntegration.config[k].type === 'string'">
                             <ng-container *ngIf="newIntegration.config[k].type === 'text'">
-                                <codemirror [(ngModel)]="newIntegration.config[k].value" [config]="codeMirrorConfig"
-                                    #codeMirror></codemirror>
+                                <codemirror [(ngModel)]="newIntegration.config[k].value" [config]="codeMirrorConfig" #codeMirror></codemirror>
                             </ng-container>
-                            <input type="password" [(ngModel)]="newIntegration.config[k].value"
-                                *ngIf="newIntegration.config[k].type === 'password'">
+                            <input type="password" [(ngModel)]="newIntegration.config[k].value" *ngIf="newIntegration.config[k].type === 'password'">
                         </div>
                     </div>
                 </ng-container>

ui/src/app/views/project/show/integrations/list/project.integration.list.html

@@ -29,6 +29,7 @@
                             </label>
                         </div>
                         <div class="ten wide field">
+                            <input type="checkbox" [(ngModel)]="p.config[k].value" *ngIf="p.config[k].type === 'boolean'" (keydown)="p.hasChanged = true" [readonly]="p.model.public"/>
                             <input type="text" [(ngModel)]="p.config[k].value" *ngIf="p.config[k].type === 'string'" (keydown)="p.hasChanged = true" [readonly]="p.model.public">
                             <ng-container *ngIf="p.config[k].type === 'text'">
                                 <codemirror [(ngModel)]="p.config[k].value" [config]="codeMirrorConfig" #codeMirror (keydown)="p.hasChanged = true"></codemirror>