From 593d3aeb88203c1e6073c1f28d6d8ee583448156 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Sun, 4 Oct 2020 23:17:01 +0200 Subject: [PATCH 1/7] fix(api): sendmail utf-8 fix #5351 Signed-off-by: Yvonnick Esnault --- engine/api/mail/mail.go | 164 ++++++++-------------------------------- go.mod | 1 + go.sum | 2 + 3 files changed, 35 insertions(+), 132 deletions(-) diff --git a/engine/api/mail/mail.go b/engine/api/mail/mail.go index 26a774e248..295d3fb5ce 100644 --- a/engine/api/mail/mail.go +++ b/engine/api/mail/mail.go @@ -4,19 +4,19 @@ import ( "bytes" "context" "crypto/tls" - "errors" "fmt" - "net/mail" "net/smtp" + "sync/atomic" "text/template" - "time" + "github.com/jordan-wright/email" "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" ) var smtpUser, smtpPassword, smtpFrom, smtpHost, smtpPort string var smtpTLS, smtpEnable bool +var lastError error +var counter uint64 const templateSignedup = `Welcome to CDS, @@ -64,69 +64,13 @@ func Init(user, password, from, host, port string, tls, disable bool) { // Status verification of smtp configuration, returns OK or KO func Status(ctx context.Context) sdk.MonitoringStatusLine { - if _, err := smtpClient(ctx); err != nil { - return sdk.MonitoringStatusLine{Component: "SMTP Ping", Value: "KO: " + err.Error(), Status: sdk.MonitoringStatusAlert} - } - return sdk.MonitoringStatusLine{Component: "SMTP Ping", Value: "Connect OK", Status: sdk.MonitoringStatusOK} -} - -func smtpClient(ctx context.Context) (*smtp.Client, error) { - if smtpHost == "" || smtpPort == "" || !smtpEnable { - return nil, errors.New("No SMTP configuration") - } - - // Connect to the SMTP Server - servername := fmt.Sprintf("%s:%s", smtpHost, smtpPort) - - // TLS config - tlsconfig := &tls.Config{ - InsecureSkipVerify: false, - ServerName: smtpHost, - } - - var c *smtp.Client - var err error - if smtpTLS { - // Here is the key, you need to call tls.Dial instead of smtp.Dial - // for smtp servers running on 465 that require an ssl connection - // from the very beginning (no starttls) - conn, err := tls.Dial("tcp", servername, tlsconfig) - if err != nil { - log.Warning(ctx, "Error with c.Dial:%s\n", err.Error()) - return nil, sdk.WithStack(err) - } - - c, err = smtp.NewClient(conn, smtpHost) - if err != nil { - log.Warning(ctx, "Error with c.NewClient:%s\n", err.Error()) - return nil, sdk.WithStack(err) - } - // TLS config - tlsconfig := &tls.Config{ - InsecureSkipVerify: false, - ServerName: smtpHost, - } - if err := c.StartTLS(tlsconfig); err != nil { - return nil, sdk.WithStack(err) - } - } else { - c, err = smtp.Dial(servername) - if err != nil { - log.Warning(ctx, "Error with c.NewClient:%s\n", err.Error()) - return nil, sdk.WithStack(err) - } + if !smtpEnable { + return sdk.MonitoringStatusLine{Component: "SMTP Ping", Value: "Conf: SMTP Disabled", Status: sdk.MonitoringStatusWarn} } - - // Auth - if smtpUser != "" && smtpPassword != "" { - auth := smtp.PlainAuth("", smtpUser, smtpPassword, smtpHost) - if err = c.Auth(auth); err != nil { - log.Warning(ctx, "Error with c.Auth:%s\n", err.Error()) - c.Close() - return nil, err - } + if lastError != nil { + return sdk.MonitoringStatusLine{Component: "SMTP Ping", Value: "KO: " + lastError.Error(), Status: sdk.MonitoringStatusAlert} } - return c, nil + return sdk.MonitoringStatusLine{Component: "SMTP Ping", Value: fmt.Sprintf("OK (%d sent)", counter), Status: sdk.MonitoringStatusOK} } // SendMailVerifyToken send mail to verify user account. @@ -172,83 +116,39 @@ func createTemplate(templ, callbackURL, callbackAPIURL, username, token string) //SendEmail is the core function to send an email func SendEmail(ctx context.Context, subject string, mailContent *bytes.Buffer, userMail string, isHTML bool) error { - from := mail.Address{ - Name: "", - Address: smtpFrom, - } - to := mail.Address{ - Name: "", - Address: userMail, - } - - // Setup headers - headers := make(map[string]string) - headers["From"] = smtpFrom - headers["To"] = to.String() - if sdk.StringIsAscii(subject) { - headers["Subject"] = subject - } else { - // https://tools.ietf.org/html/rfc2047 - headers["Subject"] = "=?UTF-8?Q?" + subject + "?=" - } - - // https://tools.ietf.org/html/rfc4021 - headers["Date"] = time.Now().Format(time.RFC1123Z) - - // https://tools.ietf.org/html/rfc2392 - headers["Message-ID"] = fmt.Sprintf("<%d.%s>", time.Now().UnixNano(), smtpFrom) - + e := email.NewEmail() + e.From = smtpFrom + e.To = []string{userMail} + e.Subject = subject + e.Text = mailContent.Bytes() if isHTML { - headers["Content-Type"] = `text/html; charset="utf-8"` - } - - // Setup message - message := "" - for k, v := range headers { - message += fmt.Sprintf("%s: %s\r\n", k, v) + e.HTML = mailContent.Bytes() } - message += "\r\n" + mailContent.String() if !smtpEnable { fmt.Println("##### NO SMTP DISPLAY MAIL IN CONSOLE ######") fmt.Printf("Subject:%s\n", subject) - fmt.Printf("Text:%s\n", message) + fmt.Printf("Text:%s\n", string(e.Text)) fmt.Println("##### END MAIL ######") return nil } - - c, err := smtpClient(ctx) - if err != nil { - return sdk.WrapError(err, "Cannot get smtp client") - } - defer c.Close() - - // To && From - if err = c.Mail(from.Address); err != nil { - return sdk.WrapError(err, "Error with c.Mail") - } - - if err = c.Rcpt(to.Address); err != nil { - return sdk.WrapError(err, "Error with c.Rcpt") - } - - // Data - w, err := c.Data() - if err != nil { - return sdk.WrapError(err, "Error with c.Data") - } - - _, err = w.Write([]byte(message)) - if err != nil { - return sdk.WrapError(err, "Error with c.Write") + servername := fmt.Sprintf("%s:%s", smtpHost, smtpPort) + auth := smtp.PlainAuth("", smtpUser, smtpPassword, smtpHost) + var err error + if smtpTLS { + tlsconfig := &tls.Config{ + InsecureSkipVerify: false, + ServerName: smtpHost, + } + err = e.SendWithStartTLS(servername, auth, tlsconfig) + } else { + err = e.Send(servername, auth) } - - err = w.Close() if err != nil { - return sdk.WrapError(err, "Error with c.Close") + lastError = err + } else { + atomic.AddUint64(&counter, 1) + lastError = nil } - - c.Quit() - - return nil + return err } diff --git a/go.mod b/go.mod index f7d2878da5..3b1c5380b8 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/itsjamie/gin-cors v0.0.0-20160420130702-97b4a9da7933 + github.com/jordan-wright/email v4.0.1-0.20200917010138-e1c00e156980+incompatible github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/keybase/go-crypto v0.0.0-20181127160227-255a5089e85a diff --git a/go.sum b/go.sum index 1009bd7cc9..f8f21cf289 100644 --- a/go.sum +++ b/go.sum @@ -256,6 +256,8 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jordan-wright/email v4.0.1-0.20200917010138-e1c00e156980+incompatible h1:CL0ooBNfbNyJTJATno+m0h+zM5bW6v7fKlboKUGP/dI= +github.com/jordan-wright/email v4.0.1-0.20200917010138-e1c00e156980+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= From afbb6533e675b71b7422fa1c821f9842705ce803 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 5 Oct 2020 09:15:12 +0200 Subject: [PATCH 2/7] fix: do not use auth when no user or password Signed-off-by: Yvonnick Esnault --- engine/api/mail/mail.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/api/mail/mail.go b/engine/api/mail/mail.go index 295d3fb5ce..4aa01b67d5 100644 --- a/engine/api/mail/mail.go +++ b/engine/api/mail/mail.go @@ -133,7 +133,10 @@ func SendEmail(ctx context.Context, subject string, mailContent *bytes.Buffer, u return nil } servername := fmt.Sprintf("%s:%s", smtpHost, smtpPort) - auth := smtp.PlainAuth("", smtpUser, smtpPassword, smtpHost) + var auth smtp.Auth + if smtpUser != "" && smtpPassword != "" { + auth = smtp.PlainAuth("", smtpUser, smtpPassword, smtpHost) + } var err error if smtpTLS { tlsconfig := &tls.Config{ From c6f49359f1c4a1d49b01c88c592655b497489da5 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 5 Oct 2020 11:42:55 +0200 Subject: [PATCH 3/7] fix: it Signed-off-by: Yvonnick Esnault --- engine/api/mail/mail.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/api/mail/mail.go b/engine/api/mail/mail.go index 4aa01b67d5..d64e26b509 100644 --- a/engine/api/mail/mail.go +++ b/engine/api/mail/mail.go @@ -27,7 +27,7 @@ To verify your email address, follow this link: If you are using the command line, you can run: -$ cdsctl signup verify --api-url {{.APIURL}} {{.Token}} +$ cdsctl signup verify --api-url {{.APIURL}} {{.Token}} Regards, -- @@ -44,7 +44,7 @@ Follow this link to set a new password on your account: If you are using the command line, you can run: -$ cdsctl reset-password confirm --api-url {{.APIURL}} {{.Token}} +$ cdsctl reset-password confirm --api-url {{.APIURL}} {{.Token}} Regards, -- From 6f3bd72817dab15b9a4e9a93d0650fc142f035fe Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 5 Oct 2020 15:51:00 +0200 Subject: [PATCH 4/7] fix: expose content decoded Signed-off-by: Yvonnick Esnault --- tests/01_signup.yml | 2 +- tests/04_sc_workflow_run_notif.yml | 2 +- tools/smtpmock/message.go | 13 +++++++------ tools/smtpmock/server/smtp.go | 11 ++++++++++- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/01_signup.yml b/tests/01_signup.yml index 71ff42e2b7..0a46902a4f 100644 --- a/tests/01_signup.yml +++ b/tests/01_signup.yml @@ -46,7 +46,7 @@ testcases: delay: 3 vars: verify: - from: result.bodyjson.content + from: result.bodyjson.content-decoded regex: cdsctl signup verify --api-url (?:.*) (.*) diff --git a/tests/04_sc_workflow_run_notif.yml b/tests/04_sc_workflow_run_notif.yml index 5d6874db3d..08a7a3a423 100644 --- a/tests/04_sc_workflow_run_notif.yml +++ b/tests/04_sc_workflow_run_notif.yml @@ -69,7 +69,7 @@ testcases: delay: 3 vars: verify: - from: result.bodyjson.content + from: result.bodyjson.content-decoded regex: logcontent:foo2 - name: run workflow 04SCWorkflowRunNotif-WORKFLOW-EMPTY diff --git a/tools/smtpmock/message.go b/tools/smtpmock/message.go index aa29ee47a9..4a706179a1 100644 --- a/tools/smtpmock/message.go +++ b/tools/smtpmock/message.go @@ -1,10 +1,11 @@ package smtpmock type Message struct { - FromAgent string `json:"from-agent"` - RemoteAddress string `json:"remote-address"` - User string `json:"user"` - From string `json:"from"` - To string `json:"to"` - Content string `json:"content"` + FromAgent string `json:"from-agent"` + RemoteAddress string `json:"remote-address"` + User string `json:"user"` + From string `json:"from"` + To string `json:"to"` + Content string `json:"content"` + ContentDecoded string `json:"content-decoded"` } diff --git a/tools/smtpmock/server/smtp.go b/tools/smtpmock/server/smtp.go index 253e3d254d..b32ac60196 100644 --- a/tools/smtpmock/server/smtp.go +++ b/tools/smtpmock/server/smtp.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "io/ioutil" + "mime/quotedprintable" + "strings" "github.com/fsamin/smtp" - + "github.com/ovh/cds/tools/smtpmock" ) @@ -33,6 +35,13 @@ func smtpHandler(envelope *smtp.Envelope) error { m.Content = string(btes) + r := quotedprintable.NewReader(strings.NewReader(m.Content)) + b, err := ioutil.ReadAll(r) + if err != nil { + return err + } + m.ContentDecoded = string(b) + StoreAddMessage(m) return nil From eb54812623218a37dc99456ee7b5c50e28a9951c Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 5 Oct 2020 15:53:18 +0200 Subject: [PATCH 5/7] fix: it Signed-off-by: Yvonnick Esnault --- tests/01_signup.yml | 2 +- tests/04_sc_workflow_run_notif.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/01_signup.yml b/tests/01_signup.yml index 0a46902a4f..d101aa84ab 100644 --- a/tests/01_signup.yml +++ b/tests/01_signup.yml @@ -46,7 +46,7 @@ testcases: delay: 3 vars: verify: - from: result.bodyjson.content-decoded + from: result.bodyjson.contentdecoded regex: cdsctl signup verify --api-url (?:.*) (.*) diff --git a/tests/04_sc_workflow_run_notif.yml b/tests/04_sc_workflow_run_notif.yml index 08a7a3a423..34f51ba473 100644 --- a/tests/04_sc_workflow_run_notif.yml +++ b/tests/04_sc_workflow_run_notif.yml @@ -69,7 +69,7 @@ testcases: delay: 3 vars: verify: - from: result.bodyjson.content-decoded + from: result.bodyjson.contentdecoded regex: logcontent:foo2 - name: run workflow 04SCWorkflowRunNotif-WORKFLOW-EMPTY From 604b9ff35786f1d465e0d925da9eb3cd8f75f28b Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 5 Oct 2020 16:12:52 +0200 Subject: [PATCH 6/7] fix: it Signed-off-by: Yvonnick Esnault --- engine/api/mail/mail.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/api/mail/mail.go b/engine/api/mail/mail.go index d64e26b509..4aa01b67d5 100644 --- a/engine/api/mail/mail.go +++ b/engine/api/mail/mail.go @@ -27,7 +27,7 @@ To verify your email address, follow this link: If you are using the command line, you can run: -$ cdsctl signup verify --api-url {{.APIURL}} {{.Token}} +$ cdsctl signup verify --api-url {{.APIURL}} {{.Token}} Regards, -- @@ -44,7 +44,7 @@ Follow this link to set a new password on your account: If you are using the command line, you can run: -$ cdsctl reset-password confirm --api-url {{.APIURL}} {{.Token}} +$ cdsctl reset-password confirm --api-url {{.APIURL}} {{.Token}} Regards, -- From 3d2bd24828b0b0eb666c5c2ab7377626ab82df86 Mon Sep 17 00:00:00 2001 From: Yvonnick Esnault Date: Mon, 5 Oct 2020 17:59:17 +0200 Subject: [PATCH 7/7] fix: it Signed-off-by: Yvonnick Esnault --- tests/01_signup.yml | 2 +- tests/04_sc_workflow_run_notif.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/01_signup.yml b/tests/01_signup.yml index d101aa84ab..0a46902a4f 100644 --- a/tests/01_signup.yml +++ b/tests/01_signup.yml @@ -46,7 +46,7 @@ testcases: delay: 3 vars: verify: - from: result.bodyjson.contentdecoded + from: result.bodyjson.content-decoded regex: cdsctl signup verify --api-url (?:.*) (.*) diff --git a/tests/04_sc_workflow_run_notif.yml b/tests/04_sc_workflow_run_notif.yml index 34f51ba473..08a7a3a423 100644 --- a/tests/04_sc_workflow_run_notif.yml +++ b/tests/04_sc_workflow_run_notif.yml @@ -69,7 +69,7 @@ testcases: delay: 3 vars: verify: - from: result.bodyjson.contentdecoded + from: result.bodyjson.content-decoded regex: logcontent:foo2 - name: run workflow 04SCWorkflowRunNotif-WORKFLOW-EMPTY