From 8ceea516769d109c5eb2e532223d1d53e4274edc Mon Sep 17 00:00:00 2001 From: Alan Sebastian Date: Thu, 3 Oct 2019 19:37:37 -0400 Subject: [PATCH] Use zap instead of logrus for the logger in v2 S3 watcher --- Gopkg.lock | 42 +++++++++++++++----- watchers/v2/s3/s3.go | 94 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 111 insertions(+), 25 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f5e5cef..aca4e0f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -199,14 +199,6 @@ revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" version = "v1.1.0" -[[projects]] - branch = "master" - digest = "1:bf3e8415ee58b09f3e544d5b464f7e026a160bbbdf6c38e156b6ec0c88356fc6" - name = "github.com/peterbourgon/mergemap" - packages = ["."] - pruneopts = "" - revision = "e21c03b7a721e413718d2703181ead350d3c9d6f" - [[projects]] digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" name = "github.com/pkg/errors" @@ -309,6 +301,37 @@ pruneopts = "" revision = "1731857f09b1f38450e2c12409748407822dc6be" +[[projects]] + digest = "1:e6ff7840319b6fda979a918a8801005ec2049abca62af19211d96971d8ec3327" + name = "go.uber.org/atomic" + packages = ["."] + pruneopts = "" + revision = "df976f2515e274675050de7b3f42545de80594fd" + version = "v1.4.0" + +[[projects]] + digest = "1:8fe76ce70e4077341d222a8a1040ac4d5ca66b23d5f58b446ba845a0c4c5884a" + name = "go.uber.org/multierr" + packages = ["."] + pruneopts = "" + revision = "71e610a0e48dbda460d9c18adca8b5f2de04a7c1" + version = "v1.2.0" + +[[projects]] + digest = "1:984e93aca9088b440b894df41f2043b6a3db8f9cf30767032770bfc4796993b0" + name = "go.uber.org/zap" + packages = [ + ".", + "buffer", + "internal/bufferpool", + "internal/color", + "internal/exit", + "zapcore", + ] + pruneopts = "" + revision = "27376062155ad36be76b0f12cf1572a221d3a48c" + version = "v1.10.0" + [[projects]] branch = "master" digest = "1:a992f0c68fa56538ede286e9e3827341fab57b7532552a771f47a32db0e6117b" @@ -368,12 +391,13 @@ "github.com/gorilla/mux", "github.com/hashicorp/consul/api", "github.com/hashicorp/consul/testutil", - "github.com/peterbourgon/mergemap", "github.com/sirupsen/logrus", "github.com/spf13/viper", "github.com/stretchr/testify/assert", "github.com/stretchr/testify/mock", "github.com/tidwall/gjson", + "go.uber.org/zap", + "go.uber.org/zap/zapcore", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/watchers/v2/s3/s3.go b/watchers/v2/s3/s3.go index 19e91ae..a4b7dea 100644 --- a/watchers/v2/s3/s3.go +++ b/watchers/v2/s3/s3.go @@ -3,7 +3,6 @@ package s3 import ( "encoding/json" "io/ioutil" - "os" "reflect" "regexp" "strings" @@ -17,7 +16,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" "github.com/rapid7/cps/pkg/index" "github.com/rapid7/cps/pkg/kv" @@ -38,6 +38,8 @@ var ( Config config isJSON = regexp.MustCompile(".json$") mu = sync.Mutex{} + + log *zap.Logger ) type config struct { @@ -46,8 +48,24 @@ type config struct { } func init() { - log.SetFormatter(&log.JSONFormatter{}) - log.SetOutput(os.Stdout) + log, _ = zap.Config{ + Encoding: "json", + Level: zap.NewAtomicLevelAt(zapcore.DebugLevel), + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: zapcore.EncoderConfig{ + MessageKey: "message", + + LevelKey: "level", + EncodeLevel: zapcore.CapitalLevelEncoder, + + TimeKey: "time", + EncodeTime: zapcore.ISO8601TimeEncoder, + + CallerKey: "caller", + EncodeCaller: zapcore.ShortCallerEncoder, + }, + }.Build() } // Poll polls every 60 seconds, kicking off an S3 sync. @@ -78,7 +96,7 @@ func Poll(bucket, bucketRegion string) { // AWS session, lists all items in the bucket, finally // parsing all files and putting them in the kv store. func Sync() { - log.Print("s3 sync begun") + log.Info("s3 sync begun") bucket := Config.bucket region := Config.bucketRegion @@ -86,7 +104,12 @@ func Sync() { svc := setUpAwsSession(region) resp, err := listBucket(bucket, region, svc) if err != nil { - log.Error(err) + log.Error("Failed to list bucket", + zap.Error(err), + zap.String("bucket", bucket), + zap.String("region", region), + ) + return } @@ -99,7 +122,7 @@ func Sync() { Up = true Health = true - log.Print("S3 sync finished") + log.Info("S3 sync finished") } func setUpAwsSession(region string) s3iface.S3API { @@ -130,8 +153,15 @@ func listBucket(bucket, region string, svc s3iface.S3API) ([]*s3.ListObjectsOutp resp, err := svc.ListObjects(params) if err != nil { - log.Errorf("Error listing s3 objects %v:", err) + log.Error("Error listing s3 objects:", + zap.Error(err), + zap.String("bucket", bucket), + zap.String("region", region), + zap.String("prefix", prefix), + ) + Health = false + return nil, err } @@ -168,7 +198,12 @@ func getPropertyFiles(files []string, b string, svc s3iface.S3API) error { serviceProperties := make(map[string]interface{}) err := json.Unmarshal(body, &serviceProperties) if err != nil { - log.Errorf("There was an error unmarshalling properties for %v: %v", serviceName, err) + log.Error("There was an error unmarshalling properties", + zap.Error(err), + zap.String("service_name", serviceName), + zap.String("file", f), + ) + return err } @@ -177,11 +212,14 @@ func getPropertyFiles(files []string, b string, svc s3iface.S3API) error { s, err := injectSecrets(services) if err != nil { - log.Errorf("%v", err) + log.Error("There was an error injecting secrets", + zap.Error(err), + zap.Any("services", services), + ) + return err } - log.Debugf("All services found: %v", s) for k, v := range s { serviceBytes, _ := json.Marshal(v) kv.WriteProperty(k, serviceBytes) @@ -207,7 +245,11 @@ func injectSecrets(data interface{}) (map[string]interface{}, error) { secretBytes, _ := json.Marshal(d.MapIndex(k).Interface()) s, err := secret.GetSSMSecret(k.String(), secretBytes) if err != nil { - log.Error(err) + log.Error("Failed to get secret from SSM", + zap.Error(err), + zap.String("key", k.String()), + ) + return nil, err } @@ -263,25 +305,45 @@ func getFile(k, b string, svc s3iface.S3API) ([]byte, error) { if err != nil { if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode { - log.Errorf("Download canceled due to timeout %v\n", err) + log.Error("Download canceled due to timeout", + zap.Error(err), + zap.String("key", k), + zap.String("bucket", b), + ) + Health = false + return nil, err } - log.Errorf("Failed to download object: %v", err) + log.Error("Failed to download object", + zap.Error(err), + zap.String("key", k), + zap.String("bucket", b), + ) + Health = false + return nil, err } body, err = ioutil.ReadAll(result.Body) defer result.Body.Close() if err != nil { - log.Errorf("Failure to read body: %v\n", err) + log.Error("Failure to read body:", + zap.Error(err), + zap.String("key", k), + zap.String("bucket", b), + ) + Health = false + return nil, err } } else { - log.Printf("Skipping: %v.\n", k) + log.Info("Skipping key", + zap.String("key", k), + ) } return body, nil