-
Notifications
You must be signed in to change notification settings - Fork 557
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add S3ObjectLambdaEvent #536
Changes from 2 commits
e6212d1
90a9fb6
0de482b
8e3bb1e
0327fe5
4a87eb4
8df4a4b
8005661
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
# Sample Function | ||
|
||
The following is a sample class and Lambda function that receives Amazon S3 Object Lambda event record data as an input and returns an object metadata output. | ||
|
||
```go | ||
|
||
// main.go | ||
package main | ||
|
||
import ( | ||
"context" | ||
"crypto/md5" | ||
"encoding/hex" | ||
"encoding/json" | ||
"io/ioutil" | ||
"net/http" | ||
"github.com/aws/aws-lambda-go/lambda" | ||
"github.com/aws/aws-lambda-go/events" | ||
"github.com/aws/aws-sdk-go-v2/config" | ||
"github.com/aws/aws-sdk-go-v2/service/s3" | ||
) | ||
|
||
func handler(ctx context.Context, event events.S3ObjectLambdaEvent) (*s3.WriteGetObjectResponseOutput, error) { | ||
url := event.GetObjectContext.InputS3Url | ||
resp, err := http.Get(url) | ||
if err != nil { | ||
return nil, err | ||
} | ||
defer resp.Body.Close() | ||
bodyBytes, err := ioutil.ReadAll(resp.Body) | ||
if err != nil { | ||
return nil, err | ||
} | ||
transformedObject := TransformedObject{ | ||
Metadata: Metadata{ | ||
Length: len(bodyBytes), | ||
Md5: toMd5(bodyBytes), | ||
}, | ||
} | ||
jsonData, err := json.Marshal(transformedObject) | ||
if err != nil { | ||
return nil, err | ||
} | ||
cfg, err := config.LoadDefaultConfig(context.TODO()) | ||
if err != nil { | ||
return nil, err | ||
} | ||
svc := s3.NewFromConfig(cfg) | ||
input := &s3.WriteGetObjectResponseInput{ | ||
RequestRoute: &event.GetObjectContext.OutputRoute, | ||
RequestToken: &event.GetObjectContext.OutputToken, | ||
Body: strings.NewReader(string(jsonData)), | ||
} | ||
return svc.WriteGetObjectResponse(ctx, input) | ||
} | ||
|
||
func toMd5(data []byte) string { | ||
hasher := md5.New() | ||
hasher.Write(data) | ||
hash := hasher.Sum(nil) | ||
|
||
return hex.EncodeToString(hash) | ||
} | ||
|
||
type TransformedObject struct { | ||
Metadata Metadata `json:"metadata"` | ||
} | ||
|
||
type Metadata struct { | ||
Length int `json:"length"` | ||
Md5 string `json:"md5"` | ||
} | ||
|
||
func main() { | ||
lambda.Start(handler) | ||
} | ||
|
||
``` |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
package events | ||
|
||
type S3ObjectLambdaEvent struct { | ||
XAmzRequestId string `json:"xAmzRequestId"` | ||
GetObjectContext *GetObjectContext `json:"getObjectContext,omitempty"` | ||
ListObjectsContext *ListObjectsContext `json:"listObjectsContext,omitempty"` | ||
ListObjectsV2Context *ListObjectsV2Context `json:"listObjectsV2Context,omitempty"` | ||
HeadObjectContext *HeadObjectContext `json:"headObjectContext,omitempty"` | ||
Configuration Configuration `json:"configuration"` | ||
UserRequest UserRequest `json:"userRequest"` | ||
UserIdentity UserIdentity `json:"userIdentity"` | ||
ProtocolVersion string `json:"protocolVersion"` | ||
} | ||
|
||
type GetObjectContext struct { | ||
InputS3Url string `json:"inputS3Url"` | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Linter should have flagged this 🧐 Should be There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks, I fixed them. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FYI: @kdnakt it looks like the README.md was not updated with this change There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for letting me know! |
||
OutputRoute string `json:"outputRoute"` | ||
OutputToken string `json:"outputToken"` | ||
} | ||
|
||
type ListObjectsContext struct { | ||
InputS3Url string `json:"inputS3Url"` | ||
} | ||
|
||
type ListObjectsV2Context struct { | ||
InputS3Url string `json:"inputS3Url"` | ||
} | ||
|
||
type HeadObjectContext struct { | ||
InputS3Url string `json:"inputS3Url"` | ||
} | ||
|
||
type Configuration struct { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. All the new structs in the PR should be prefixed with something like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the suggestion, I added the prefix. |
||
AccessPointArn string `json:"accessPointArn"` | ||
SupportingAccessPointArn string `json:"supportingAccessPointArn"` | ||
Payload string `json:"payload"` | ||
} | ||
|
||
type UserRequest struct { | ||
Url string `json:"url"` | ||
Headers map[string]string `json:"headers"` | ||
} | ||
|
||
type UserIdentity struct { | ||
Type string `json:"type"` | ||
PrincipalId string `json:"principalId"` | ||
Arn string `json:"arn"` | ||
AccountId string `json:"accountId"` | ||
AccessKeyId string `json:"accessKeyId"` | ||
SessionContext *SessionContext `json:"sessionContext,omitempty"` | ||
} | ||
|
||
type SessionContext struct { | ||
Attributes map[string]string `json:"attributes"` | ||
SessionIssuer *SessionIssuer `json:"sessionIssuer,omitempty"` | ||
} | ||
|
||
type SessionIssuer struct { | ||
Type string `json:"type"` | ||
PrincipalId string `json:"principalId"` | ||
Arn string `json:"arn"` | ||
AccountId string `json:"accountId"` | ||
UserName string `json:"userName"` | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
package events | ||
|
||
import ( | ||
"encoding/json" | ||
"testing" | ||
|
||
"github.com/aws/aws-lambda-go/events/test" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestS3ObjectLambdaEventMarshaling(t *testing.T) { | ||
tests := []struct { | ||
file string | ||
}{ | ||
{"./testdata/s3-object-lambda-event-get-object-iam.json"}, | ||
{"./testdata/s3-object-lambda-event-get-object-assumed-role.json"}, | ||
{"./testdata/s3-object-lambda-event-head-object-iam.json"}, | ||
{"./testdata/s3-object-lambda-event-list-objects-iam.json"}, | ||
{"./testdata/s3-object-lambda-event-list-objects-v2-iam.json"}, | ||
} | ||
|
||
for _, tc := range tests { | ||
tc := tc | ||
t.Run(tc.file, func(t *testing.T) { | ||
inputJSON := test.ReadJSONFromFile(t, tc.file) | ||
|
||
var inputEvent S3ObjectLambdaEvent | ||
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil { | ||
t.Errorf("could not unmarshal event. details: %v", err) | ||
} | ||
|
||
outputJSON, err := json.Marshal(inputEvent) | ||
if err != nil { | ||
t.Errorf("could not marshal event. details: %v", err) | ||
} | ||
|
||
assert.JSONEq(t, string(inputJSON), string(outputJSON)) | ||
}) | ||
} | ||
} | ||
|
||
func TestS3ObjectLambdaMarshalingMalformedJson(t *testing.T) { | ||
test.TestMalformedJson(t, S3ObjectLambdaEvent{}) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"getObjectContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>", | ||
"outputRoute": "io-use1-001", | ||
"outputToken": "OutputToken" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "AssumedRole", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:sts::111122223333:assumed-role/Admin/example", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId", | ||
"sessionContext": { | ||
"attributes": { | ||
"mfaAuthenticated": "false", | ||
"creationDate": "Wed Mar 10 23:41:52 UTC 2021" | ||
}, | ||
"sessionIssuer": { | ||
"type": "Role", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:role/Admin", | ||
"accountId": "111122223333", | ||
"userName": "Admin" | ||
} | ||
} | ||
}, | ||
"protocolVersion": "1.00" | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"getObjectContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>", | ||
"outputRoute": "io-use1-001", | ||
"outputToken": "OutputToken" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.00" | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"headObjectContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.01" | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"listObjectsContext": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.01" | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
{ | ||
"xAmzRequestId": "requestId", | ||
"listObjectsV2Context": { | ||
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>" | ||
}, | ||
"configuration": { | ||
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", | ||
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", | ||
"payload": "{}" | ||
}, | ||
"userRequest": { | ||
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", | ||
"headers": { | ||
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", | ||
"Accept-Encoding": "identity", | ||
"X-Amz-Content-SHA256": "e3b0c44298fc1example" | ||
} | ||
}, | ||
"userIdentity": { | ||
"type": "IAMUser", | ||
"principalId": "principalId", | ||
"arn": "arn:aws:iam::111122223333:user/username", | ||
"accountId": "111122223333", | ||
"accessKeyId": "accessKeyId" | ||
}, | ||
"protocolVersion": "1.01" | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
At a glance this doesn't look right as an example return value.
Looking at the docs https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-writing-lambda.html#olap-headobject
I see python functions returning things like
{statusCode: 200, <otherfields>}
and at least on java function defined with avoid
return type.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Changed this function just to return error.