fix(ui,api,plugin:clair): save and render vulnerabilities, clair panic (

#5234)

* fix(plugin:clair,ui,api): save and render vulnerabilities, clair panic

* chore: exclude windows target for clair plugin

contrib/grpcplugins/action/plugin-clair/go.mod

@@ -4,9 +4,9 @@ go 1.14
 
 replace github.com/ovh/cds => ../../../../
 
-replace github.com/docker/docker => github.com/docker/engine v0.0.0-20180816081446-320063a2ad06
+replace github.com/docker/docker => github.com/moby/moby v17.12.0-ce-rc1.0.20200528182317-b47e74255811+incompatible
 
-replace github.com/Sirupsen/logrus v1.4.0 => github.com/sirupsen/logrus v1.4.0
+replace github.com/Sirupsen/logrus => github.com/sirupsen/logrus v1.6.0
 
 replace github.com/codegangsta/cli v1.22.2 => github.com/urfave/cli v1.22.2
 
@@ -15,24 +15,28 @@ replace github.com/prometheus/client_golang v1.1.0 => github.com/prometheus/clie
 replace github.com/opencontainers/runc v0.1.1 => github.com/opencontainers/runc v1.0.0-rc9
 
 require (
-	github.com/containerd/containerd v1.3.2 // indirect
-	github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb // indirect
+	github.com/Microsoft/hcsshim v0.8.9 // indirect
+	github.com/Sirupsen/logrus v0.0.0-00010101000000-000000000000 // indirect
+	github.com/containerd/containerd v1.3.4 // indirect
+	github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
-	github.com/golang/protobuf v1.3.2
+	github.com/golang/protobuf v1.4.2
 	github.com/jgsqware/xnet v0.0.0-20170203143001-13630f0737d2
-	github.com/mattn/go-shellwords v1.0.6 // indirect
+	github.com/mattn/go-shellwords v1.0.10 // indirect
 	github.com/mholt/archiver v3.1.1+incompatible
-	github.com/opencontainers/go-digest v1.0.0-rc1
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/runc v0.1.1 // indirect
-	github.com/opencontainers/runtime-spec v1.0.1 // indirect
-	github.com/opencontainers/selinux v1.3.0 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/opencontainers/selinux v1.5.2 // indirect
 	github.com/ovh/cds v0.0.0-00010101000000-000000000000
 	github.com/prometheus/client_golang v1.1.0
-	github.com/quay/clair/v2 v2.1.1
-	github.com/spf13/viper v1.5.0
+	github.com/quay/clair/v2 v2.1.4
+	github.com/spf13/viper v1.7.0
+	github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect
 	github.com/vbatts/tar-split v0.11.1 // indirect
-	golang.org/x/net v0.0.0-20191204025024-5ee1b9f4859a
+	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
 )

contrib/grpcplugins/action/plugin-clair/main.go

@@ -67,7 +67,6 @@ func (actPlugin *clairActionPlugin) Run(ctx context.Context, q *actionplugin.Act
 	fmt.Printf("Creating report")
 
 	var vulnerabilities []sdk.Vulnerability
-	summary := make(map[string]int64)
 	if analysis.MostRecentLayer().Layer != nil {
 		for _, feat := range analysis.MostRecentLayer().Layer.Features {
 			for _, vuln := range feat.Vulnerabilities {
@@ -83,16 +82,12 @@ func (actPlugin *clairActionPlugin) Run(ctx context.Context, q *actionplugin.Act
 					Title:       fmt.Sprintf("%s %s", feat.Name, feat.Version),
 				}
 				vulnerabilities = append(vulnerabilities, v)
-
-				count := summary[v.Severity]
-				summary[v.Severity] = count + 1
 			}
 		}
 	}
 
 	report := sdk.VulnerabilityWorkerReport{
 		Vulnerabilities: vulnerabilities,
-		Summary:         summary,
 		Type:            "docker",
 	}
 	if err := grpcplugins.SendVulnerabilityReport(actPlugin.HTTPPort, report); err != nil {

contrib/grpcplugins/action/plugin-npm-audit-parser/main.go

@@ -46,7 +46,7 @@ func (actPlugin *npmAuditParserActionPlugin) Run(ctx context.Context, q *actionp
 	}
 
 	var report sdk.VulnerabilityWorkerReport
-	summary := make(map[string]int64)
+
 	for _, a := range npmAudit.Advisories {
 		for _, f := range a.Findings {
 			if len(a.CVES) > 0 {
@@ -63,8 +63,6 @@ func (actPlugin *npmAuditParserActionPlugin) Run(ctx context.Context, q *actionp
 						Version:     f.Version,
 					}
 					report.Vulnerabilities = append(report.Vulnerabilities, v)
-					count := summary[v.Severity]
-					summary[v.Severity] = count + 1
 				}
 			} else {
 				v := sdk.Vulnerability{
@@ -79,14 +77,10 @@ func (actPlugin *npmAuditParserActionPlugin) Run(ctx context.Context, q *actionp
 					Version:     f.Version,
 				}
 				report.Vulnerabilities = append(report.Vulnerabilities, v)
-				count := summary[v.Severity]
-				summary[v.Severity] = count + 1
 			}
-
 		}
 	}
 	report.Type = "js"
-	report.Summary = summary
 	if err := grpcplugins.SendVulnerabilityReport(actPlugin.HTTPPort, report); err != nil {
 		return actionplugin.Fail("Unable to send report: %s", err)
 	}

engine/api/application/application_vunerability.go

@@ -12,9 +12,9 @@ import (
 // LoadVulnerabilitiesSummary compute vulnerabilities summary
 func LoadVulnerabilitiesSummary(db gorp.SqlExecutor, appID int64) (map[string]float64, error) {
 	query := `
-    SELECT json_object_agg(severity, nb)::TEXT 
+    SELECT json_object_agg(severity, nb)::TEXT
     FROM (
-	    SELECT count(id) AS nb, severity 
+	    SELECT count(id) AS nb, severity
       FROM application_vulnerability
 	    WHERE application_id = $1
 	    GROUP BY severity
@@ -24,42 +24,52 @@ func LoadVulnerabilitiesSummary(db gorp.SqlExecutor, appID int64) (map[string]fl
 	var summary map[string]float64
 	var result sql.NullString
 	if err := db.QueryRow(query, appID).Scan(&result); err != nil {
-		return nil, sdk.WrapError(err, "LoadVulnerabilitiesSummary")
+		return nil, sdk.WithStack(err)
 	}
 
 	if err := gorpmapping.JSONNullString(result, &summary); err != nil {
-		return nil, sdk.WrapError(err, "Unable to unmarshal summary")
+		return nil, sdk.WrapError(err, "unable to unmarshal summary")
 	}
 	return summary, nil
 }
 
 // InsertVulnerabilities Insert vulnerabilities
-func InsertVulnerabilities(db gorp.SqlExecutor, vs []sdk.Vulnerability, appID int64, t string) error {
-	if _, err := db.Exec("DELETE FROM application_vulnerability WHERE application_id = $1 AND type = $2", appID, t); err != nil {
-		return sdk.WrapError(err, "Unable to remove old vulnerabilities")
-	}
+func InsertVulnerabilities(db gorp.SqlExecutor, vs []sdk.Vulnerability, appID int64) error {
 	for _, v := range vs {
 		v.ApplicationID = appID
-		v.Type = t
 		dbVuln := dbApplicationVulnerability(v)
 		if err := db.Insert(&dbVuln); err != nil {
-			return sdk.WrapError(err, "Unable to insert vulnerabilities")
+			return sdk.WrapError(err, "unable to insert vulnerabilities")
 		}
 	}
 	return nil
 }
 
+// DeleteVulnerabilitiesByApplicationIDAndType removes all the vulnerabilities for given application and type.
+func DeleteVulnerabilitiesByApplicationIDAndType(db gorp.SqlExecutor, applicationID int64, vulnerabilityType string) error {
+	if _, err := db.Exec(`
+    DELETE FROM application_vulnerability
+    WHERE application_id = $1
+    AND type = $2
+  `, applicationID, vulnerabilityType); err != nil {
+		return sdk.WrapError(err, "unable to remove vulnerabilities with type %s for application with id %d", vulnerabilityType, applicationID)
+	}
+	return nil
+}
+
 // LoadVulnerabilities load vulnerabilities for the given application
 func LoadVulnerabilities(db gorp.SqlExecutor, appID int64) ([]sdk.Vulnerability, error) {
 	results := make([]dbApplicationVulnerability, 0)
-	query := `SELECT *
-            FROM application_vulnerability
-            WHERE application_id = $1`
+	query := `
+    SELECT *
+    FROM application_vulnerability
+    WHERE application_id = $1
+  `
 	if _, err := db.Select(&results, query, appID); err != nil {
 		if err != sql.ErrNoRows {
 			return nil, sdk.WrapError(err, "unable to load latest vulnerabilities for application %d", appID)
 		}
-		return nil, sdk.WithStack(sdk.ErrNotFound)
+		return nil, nil
 	}
 	vulnerabilities := make([]sdk.Vulnerability, len(results))
 	for i := range results {

engine/api/application/dao_dependencies.go

@@ -73,8 +73,8 @@ var (
 	loadVulnerabilities = func(db gorp.SqlExecutor, app *sdk.Application) error {
 		var err error
 		app.Vulnerabilities, err = LoadVulnerabilities(db, app.ID)
-		if err != nil && sdk.Cause(err) != sql.ErrNoRows {
-			return sdk.WrapError(err, "Unable to load vulnerabilities")
+		if err != nil {
+			return sdk.WrapError(err, "unable to load vulnerabilities")
 		}
 		return nil
 	}

engine/api/application_vulnerability_test.go

@@ -1,16 +1,15 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
-	"bytes"
-	"fmt"
-
 	"github.com/ovh/cds/engine/api/application"
 	"github.com/ovh/cds/engine/api/test"
 	"github.com/ovh/cds/engine/api/test/assets"
@@ -37,7 +36,7 @@ func Test_postVulnerabilityHandler(t *testing.T) {
 	v := sdk.Vulnerability{}
 	v.ApplicationID = app.ID
 
-	assert.NoError(t, application.InsertVulnerabilities(db, []sdk.Vulnerability{v}, app.ID, "docker"))
+	assert.NoError(t, application.InsertVulnerabilities(db, []sdk.Vulnerability{v}, app.ID))
 
 	vulns, err := application.LoadVulnerabilities(db, app.ID)
 	assert.NoError(t, err)
@@ -62,5 +61,4 @@ func Test_postVulnerabilityHandler(t *testing.T) {
 	w := httptest.NewRecorder()
 	router.Mux.ServeHTTP(w, req)
 	assert.Equal(t, 200, w.Code)
-
 }

engine/api/workflow/dao_node_run.go

@@ -107,14 +107,15 @@ func LoadNodeRun(db gorp.SqlExecutor, projectkey, workflowname string, number, i
 		r.Coverage = cov
 	}
 	if loadOpts.WithVulnerabilities {
-		vuln, errV := loadVulnerabilityReport(db, r.ID)
-		if errV != nil && !sdk.ErrorIs(errV, sdk.ErrNotFound) {
-			return nil, sdk.WrapError(errV, "LoadNodeRun>Error vulnerability report coverage for run %d", r.ID)
+		vuln, err := loadVulnerabilityReport(db, r.ID)
+		if err != nil && !sdk.ErrorIs(err, sdk.ErrNotFound) {
+			return nil, sdk.WrapError(err, "vulnerability report coverage for run %d", r.ID)
+		}
+		if vuln != nil {
+			r.VulnerabilitiesReport = *vuln
 		}
-		r.VulnerabilitiesReport = vuln
 	}
 	return r, nil
-
 }
 
 //LoadNodeRunByNodeJobID load a specific node run on a workflow from a node job run id