From 486954c332e7e0d839ca7d55739face48e215900 Mon Sep 17 00:00:00 2001 From: Gaius Date: Mon, 27 Jun 2022 16:26:54 +0800 Subject: [PATCH] feat: GetObjectMetadata return isExist Signed-off-by: Gaius --- api/manager/docs.go | 136 +++++++++++++------ api/manager/swagger.json | 136 +++++++++++++------ api/manager/swagger.yaml | 92 +++++++++---- client/daemon/objectstorage/objectstorage.go | 21 ++- pkg/objectstorage/objectstorage.go | 2 +- pkg/objectstorage/oss.go | 16 ++- pkg/objectstorage/s3.go | 22 +-- 7 files changed, 296 insertions(+), 129 deletions(-) diff --git a/api/manager/docs.go b/api/manager/docs.go index 44913ae4611..f5cd7c98e72 100644 --- a/api/manager/docs.go +++ b/api/manager/docs.go @@ -3692,9 +3692,15 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "download_rate_limit": { "type": "integer" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3713,6 +3719,9 @@ const docTemplate = `{ "state": { "type": "string" }, + "updated_at": { + "type": "string" + }, "url": { "type": "string" }, @@ -3724,52 +3733,22 @@ const docTemplate = `{ } } }, - "model.Assertion": { + "model.Config": { "type": "object", "properties": { - "key": { + "bio": { "type": "string" }, - "policy": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "policyMap": { - "type": "object", - "additionalProperties": { - "type": "integer" - } + "created_at": { + "type": "string" }, - "rm": {}, - "tokens": { - "type": "array", - "items": { - "type": "string" - } + "id": { + "type": "integer" }, - "value": { - "type": "string" - } - } - }, - "model.AssertionMap": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/model.Assertion" - } - }, - "model.Config": { - "type": "object", - "properties": { - "bio": { + "name": { "type": "string" }, - "name": { + "updated_at": { "type": "string" }, "user_id": { @@ -3793,6 +3772,12 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "result": { "$ref": "#/definitions/model.JSONMap" }, @@ -3817,6 +3802,9 @@ const docTemplate = `{ "type": { "type": "string" }, + "updated_at": { + "type": "string" + }, "user_id": { "type": "integer" } @@ -3834,20 +3822,35 @@ const docTemplate = `{ "client_secret": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, "redirect_url": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, "model.Scheduler": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3868,6 +3871,9 @@ const docTemplate = `{ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -3886,6 +3892,12 @@ const docTemplate = `{ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -3909,6 +3921,9 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/model.SeedPeerCluster" } + }, + "updated_at": { + "type": "string" } } }, @@ -3918,6 +3933,12 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3926,6 +3947,9 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/model.SecurityRule" } + }, + "updated_at": { + "type": "string" } } }, @@ -3935,9 +3959,15 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "domain": { "type": "string" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3949,18 +3979,27 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/model.SecurityGroup" } + }, + "updated_at": { + "type": "string" } } }, "model.SeedPeer": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "download_port": { "type": "integer" }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3987,6 +4026,9 @@ const docTemplate = `{ }, "type": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -4002,6 +4044,12 @@ const docTemplate = `{ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -4025,6 +4073,9 @@ const docTemplate = `{ }, "security_group_id": { "type": "integer" + }, + "updated_at": { + "type": "string" } } }, @@ -4037,9 +4088,15 @@ const docTemplate = `{ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "email": { "type": "string" }, + "id": { + "type": "integer" + }, "location": { "type": "string" }, @@ -4051,6 +4108,9 @@ const docTemplate = `{ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, diff --git a/api/manager/swagger.json b/api/manager/swagger.json index 41a4ac06652..7e303913021 100644 --- a/api/manager/swagger.json +++ b/api/manager/swagger.json @@ -3685,9 +3685,15 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "download_rate_limit": { "type": "integer" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3706,6 +3712,9 @@ "state": { "type": "string" }, + "updated_at": { + "type": "string" + }, "url": { "type": "string" }, @@ -3717,52 +3726,22 @@ } } }, - "model.Assertion": { + "model.Config": { "type": "object", "properties": { - "key": { + "bio": { "type": "string" }, - "policy": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "policyMap": { - "type": "object", - "additionalProperties": { - "type": "integer" - } + "created_at": { + "type": "string" }, - "rm": {}, - "tokens": { - "type": "array", - "items": { - "type": "string" - } + "id": { + "type": "integer" }, - "value": { - "type": "string" - } - } - }, - "model.AssertionMap": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/model.Assertion" - } - }, - "model.Config": { - "type": "object", - "properties": { - "bio": { + "name": { "type": "string" }, - "name": { + "updated_at": { "type": "string" }, "user_id": { @@ -3786,6 +3765,12 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "result": { "$ref": "#/definitions/model.JSONMap" }, @@ -3810,6 +3795,9 @@ "type": { "type": "string" }, + "updated_at": { + "type": "string" + }, "user_id": { "type": "integer" } @@ -3827,20 +3815,35 @@ "client_secret": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, "redirect_url": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, "model.Scheduler": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3861,6 +3864,9 @@ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -3879,6 +3885,12 @@ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -3902,6 +3914,9 @@ "items": { "$ref": "#/definitions/model.SeedPeerCluster" } + }, + "updated_at": { + "type": "string" } } }, @@ -3911,6 +3926,12 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3919,6 +3940,9 @@ "items": { "$ref": "#/definitions/model.SecurityRule" } + }, + "updated_at": { + "type": "string" } } }, @@ -3928,9 +3952,15 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "domain": { "type": "string" }, + "id": { + "type": "integer" + }, "name": { "type": "string" }, @@ -3942,18 +3972,27 @@ "items": { "$ref": "#/definitions/model.SecurityGroup" } + }, + "updated_at": { + "type": "string" } } }, "model.SeedPeer": { "type": "object", "properties": { + "created_at": { + "type": "string" + }, "download_port": { "type": "integer" }, "host_name": { "type": "string" }, + "id": { + "type": "integer" + }, "idc": { "type": "string" }, @@ -3980,6 +4019,9 @@ }, "type": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, @@ -3995,6 +4037,12 @@ "config": { "$ref": "#/definitions/model.JSONMap" }, + "created_at": { + "type": "string" + }, + "id": { + "type": "integer" + }, "is_default": { "type": "boolean" }, @@ -4018,6 +4066,9 @@ }, "security_group_id": { "type": "integer" + }, + "updated_at": { + "type": "string" } } }, @@ -4030,9 +4081,15 @@ "bio": { "type": "string" }, + "created_at": { + "type": "string" + }, "email": { "type": "string" }, + "id": { + "type": "integer" + }, "location": { "type": "string" }, @@ -4044,6 +4101,9 @@ }, "state": { "type": "string" + }, + "updated_at": { + "type": "string" } } }, diff --git a/api/manager/swagger.yaml b/api/manager/swagger.yaml index 9a59e13c1ef..4716cd5e219 100644 --- a/api/manager/swagger.yaml +++ b/api/manager/swagger.yaml @@ -4,8 +4,12 @@ definitions: properties: bio: type: string + created_at: + type: string download_rate_limit: type: integer + id: + type: integer name: type: string scheduler_clusters: @@ -18,6 +22,8 @@ definitions: type: array state: type: string + updated_at: + type: string url: type: string user: @@ -25,38 +31,18 @@ definitions: user_id: type: integer type: object - model.Assertion: - properties: - key: - type: string - policy: - items: - items: - type: string - type: array - type: array - policyMap: - additionalProperties: - type: integer - type: object - rm: {} - tokens: - items: - type: string - type: array - value: - type: string - type: object - model.AssertionMap: - additionalProperties: - $ref: '#/definitions/model.Assertion' - type: object model.Config: properties: bio: type: string + created_at: + type: string + id: + type: integer name: type: string + updated_at: + type: string user_id: type: integer value: @@ -71,6 +57,10 @@ definitions: $ref: '#/definitions/model.JSONMap' bio: type: string + created_at: + type: string + id: + type: integer result: $ref: '#/definitions/model.JSONMap' scheduler_clusters: @@ -87,6 +77,8 @@ definitions: type: string type: type: string + updated_at: + type: string user_id: type: integer type: object @@ -98,15 +90,25 @@ definitions: type: string client_secret: type: string + created_at: + type: string + id: + type: integer name: type: string redirect_url: type: string + updated_at: + type: string type: object model.Scheduler: properties: + created_at: + type: string host_name: type: string + id: + type: integer idc: type: string ip: @@ -121,6 +123,8 @@ definitions: type: integer state: type: string + updated_at: + type: string type: object model.SchedulerCluster: properties: @@ -132,6 +136,10 @@ definitions: $ref: '#/definitions/model.JSONMap' config: $ref: '#/definitions/model.JSONMap' + created_at: + type: string + id: + type: integer is_default: type: boolean jobs: @@ -148,24 +156,36 @@ definitions: items: $ref: '#/definitions/model.SeedPeerCluster' type: array + updated_at: + type: string type: object model.SecurityGroup: properties: bio: type: string + created_at: + type: string + id: + type: integer name: type: string security_rules: items: $ref: '#/definitions/model.SecurityRule' type: array + updated_at: + type: string type: object model.SecurityRule: properties: bio: type: string + created_at: + type: string domain: type: string + id: + type: integer name: type: string proxy_domain: @@ -174,13 +194,19 @@ definitions: items: $ref: '#/definitions/model.SecurityGroup' type: array + updated_at: + type: string type: object model.SeedPeer: properties: + created_at: + type: string download_port: type: integer host_name: type: string + id: + type: integer idc: type: string ip: @@ -199,6 +225,8 @@ definitions: type: string type: type: string + updated_at: + type: string type: object model.SeedPeerCluster: properties: @@ -208,6 +236,10 @@ definitions: type: string config: $ref: '#/definitions/model.JSONMap' + created_at: + type: string + id: + type: integer is_default: type: boolean jobs: @@ -224,6 +256,8 @@ definitions: $ref: '#/definitions/model.JSONMap' security_group_id: type: integer + updated_at: + type: string type: object model.User: properties: @@ -231,8 +265,12 @@ definitions: type: string bio: type: string + created_at: + type: string email: type: string + id: + type: integer location: type: string name: @@ -241,6 +279,8 @@ definitions: type: string state: type: string + updated_at: + type: string type: object objectstorage.BucketMetadata: properties: diff --git a/client/daemon/objectstorage/objectstorage.go b/client/daemon/objectstorage/objectstorage.go index 88175eca4bc..3dc432d3a85 100644 --- a/client/daemon/objectstorage/objectstorage.go +++ b/client/daemon/objectstorage/objectstorage.go @@ -191,13 +191,19 @@ func (o *objectStorage) getObject(ctx *gin.Context) { err error ) + // Initialize filter field. + urlMeta := &base.UrlMeta{Filter: o.config.ObjectStorage.Filter} + if filter != "" { + urlMeta.Filter = filter + } + client, err := o.client() if err != nil { ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) return } - isExist, err := client.IsObjectExist(ctx, bucketName, objectKey) + meta, isExist, err := client.GetObjectMetadata(ctx, bucketName, objectKey) if err != nil { ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) return @@ -208,11 +214,7 @@ func (o *objectStorage) getObject(ctx *gin.Context) { return } - // Initialize filter field. - urlMeta := &base.UrlMeta{Filter: o.config.ObjectStorage.Filter} - if filter != "" { - urlMeta.Filter = filter - } + urlMeta.Digest = meta.Digest // Parse http range header. rangeHeader := ctx.GetHeader(headers.Range) @@ -228,13 +230,6 @@ func (o *objectStorage) getObject(ctx *gin.Context) { urlMeta.Range = strings.TrimLeft(rangeHeader, "bytes=") } - meta, err := client.GetObjectMetadata(ctx, bucketName, objectKey) - if err != nil { - ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) - return - } - urlMeta.Digest = meta.Digest - signURL, err := client.GetSignURL(ctx, bucketName, objectKey, objectstorage.MethodGet, defaultSignExpireTime) if err != nil { ctx.JSON(http.StatusInternalServerError, gin.H{"errors": err.Error()}) diff --git a/pkg/objectstorage/objectstorage.go b/pkg/objectstorage/objectstorage.go index 17e6c9f11c7..af471f5b335 100644 --- a/pkg/objectstorage/objectstorage.go +++ b/pkg/objectstorage/objectstorage.go @@ -73,7 +73,7 @@ type ObjectStorage interface { ListBucketMetadatas(ctx context.Context) ([]*BucketMetadata, error) // GetObjectMetadata returns metadata of object. - GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, error) + GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, bool, error) // GetOject returns data of object. GetOject(ctx context.Context, bucketName, objectKey string) (io.ReadCloser, error) diff --git a/pkg/objectstorage/oss.go b/pkg/objectstorage/oss.go index 353151550a2..bf0273a9917 100644 --- a/pkg/objectstorage/oss.go +++ b/pkg/objectstorage/oss.go @@ -20,11 +20,13 @@ import ( "context" "fmt" "io" + "net/http" "strconv" "time" aliyunoss "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-http-utils/headers" + "github.com/pkg/errors" ) type oss struct { @@ -86,20 +88,24 @@ func (o *oss) ListBucketMetadatas(ctx context.Context) ([]*BucketMetadata, error } // GetObjectMetadata returns metadata of object. -func (o *oss) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, error) { +func (o *oss) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, bool, error) { bucket, err := o.client.Bucket(bucketName) if err != nil { - return nil, err + return nil, false, err } header, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { - return nil, err + if serr, ok := errors.Cause(err).(aliyunoss.ServiceError); ok && serr.StatusCode == http.StatusNotFound { + return nil, false, nil + } + + return nil, false, err } contentLength, err := strconv.ParseInt(header.Get(headers.ContentLength), 10, 64) if err != nil { - return nil, err + return nil, false, err } return &ObjectMetadata{ @@ -111,7 +117,7 @@ func (o *oss) GetObjectMetadata(ctx context.Context, bucketName, objectKey strin ContentType: header.Get(headers.ContentType), Etag: header.Get(headers.ETag), Digest: header.Get(aliyunoss.HTTPHeaderOssMetaPrefix + MetaDigest), - }, nil + }, true, nil } // GetOject returns data of object. diff --git a/pkg/objectstorage/s3.go b/pkg/objectstorage/s3.go index f3c49499747..ecfe38f908a 100644 --- a/pkg/objectstorage/s3.go +++ b/pkg/objectstorage/s3.go @@ -91,13 +91,18 @@ func (s *s3) ListBucketMetadatas(ctx context.Context) ([]*BucketMetadata, error) } // GetObjectMetadata returns metadata of object. -func (s *s3) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, error) { +func (s *s3) GetObjectMetadata(ctx context.Context, bucketName, objectKey string) (*ObjectMetadata, bool, error) { resp, err := s.client.HeadObjectWithContext(ctx, &awss3.HeadObjectInput{ Bucket: aws.String(bucketName), Key: aws.String(objectKey), }) if err != nil { - return nil, err + // S3 is missing this error code. + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { + return nil, false, nil + } + + return nil, false, err } return &ObjectMetadata{ @@ -109,7 +114,7 @@ func (s *s3) GetObjectMetadata(ctx context.Context, bucketName, objectKey string ContentType: aws.StringValue(resp.ContentType), Etag: aws.StringValue(resp.ETag), Digest: aws.StringValue(resp.Metadata[MetaDigest]), - }, nil + }, true, nil } // GetOject returns data of object. @@ -175,15 +180,16 @@ func (s *s3) ListObjectMetadatas(ctx context.Context, bucketName, prefix, marker // IsObjectExist returns whether the object exists. func (s *s3) IsObjectExist(ctx context.Context, bucketName, objectKey string) (bool, error) { - if _, err := s.GetObjectMetadata(ctx, bucketName, objectKey); err != nil { - // S3 is missing this error code. - if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { - return false, nil - } + _, isExist, err := s.GetObjectMetadata(ctx, bucketName, objectKey) + if err != nil { return false, err } + if !isExist { + return false, nil + } + return true, nil }