Full support of registry v2 and Clair v2 (#59)

* Remove vnd.docker.distribution.manifest.list.v2+json content type

* Add utils package for debug purposes

* Fix layer order for V2 schema

* Simple tracing, some refactoring

clair/clair.go

@@ -1,23 +1,25 @@
 package clair
 
 import (
-	"os"
 	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/optiopay/klar/docker"
+	"github.com/optiopay/klar/utils"
 )
 
 const EMPTY_LAYER_BLOB_SUM = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
 
 // Clair is representation of Clair server
 type Clair struct {
-	url string
+	url    string
+	client http.Client
 }
 
 type layer struct {
@@ -75,17 +77,21 @@ func NewClair(url string) Clair {
 	if strings.LastIndex(url, ":") < 5 {
 		url = fmt.Sprintf("%s:6060", url)
 	}
-	return Clair{url}
+	client := http.Client{
+		Timeout: time.Minute,
+	}
+
+	return Clair{url, client}
 }
 
 func newLayer(image *docker.Image, index int) *layer {
 	var parentName string
-	if index < len(image.FsLayers)-1 {
-		parentName = image.FsLayers[index+1].BlobSum
+	if index != 0 {
+		parentName = image.LayerName(index - 1)
 	}
 
 	return &layer{
-		Name:       image.FsLayers[index].BlobSum,
+		Name:       image.LayerName(index),
 		Path:       strings.Join([]string{image.Registry, image.Name, "blobs", image.FsLayers[index].BlobSum}, "/"),
 		ParentName: parentName,
 		Format:     "Docker",
@@ -115,7 +121,7 @@ func (c *Clair) Analyse(image *docker.Image) []Vulnerability {
 	}
 
 	var vs []Vulnerability
-	for i := layerLength - 1; i >= 0; i-- {
+	for i := 0; i < layerLength; i++ {
 		layer := newLayer(image, i)
 		err := c.pushLayer(layer)
 		if err != nil {
@@ -124,7 +130,7 @@ func (c *Clair) Analyse(image *docker.Image) []Vulnerability {
 		}
 	}
 
-	vs, err := c.analyzeLayer(image.FsLayers[0])
+	vs, err := c.analyzeLayer(image.AnalyzedLayerName())
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Analyse image %s/%s:%s failed: %s\n", image.Registry, image.Name, image.Tag, err.Error())
 		return nil
@@ -133,12 +139,18 @@ func (c *Clair) Analyse(image *docker.Image) []Vulnerability {
 	return vs
 }
 
-func (c *Clair) analyzeLayer(layer docker.FsLayer) ([]Vulnerability, error) {
-	url := fmt.Sprintf("%s/v1/layers/%s?vulnerabilities", c.url, layer.BlobSum)
-	response, err := http.Get(url)
+func (c *Clair) analyzeLayer(layerName string) ([]Vulnerability, error) {
+	url := fmt.Sprintf("%s/v1/layers/%s?vulnerabilities", c.url, layerName)
+	request, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("Can't create an analyze request: %s", err)
+	}
+	utils.DumpRequest(request)
+	response, err := c.client.Do(request)
 	if err != nil {
 		return nil, err
 	}
+	utils.DumpResponse(response)
 	defer response.Body.Close()
 	if response.StatusCode != http.StatusOK {
 		body, _ := ioutil.ReadAll(response.Body)
@@ -169,11 +181,12 @@ func (c *Clair) pushLayer(layer *layer) error {
 		return fmt.Errorf("Can't create a push request: %s", err)
 	}
 	request.Header.Set("Content-Type", "application/json")
-	//fmt.Printf("Pushing layer %v\n", layer)
-	response, err := (&http.Client{Timeout: time.Minute}).Do(request)
+	utils.DumpRequest(request)
+	response, err := c.client.Do(request)
 	if err != nil {
 		return fmt.Errorf("Can't push layer to Clair: %s", err)
 	}
+	utils.DumpResponse(response)
 	defer response.Body.Close()
 	body, err := ioutil.ReadAll(response.Body)
 	if err != nil {

docker/docker.go

@@ -7,10 +7,12 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"net/http/httputil"
 	"os"
 	"regexp"
 	"strings"
+	"time"
+
+	"github.com/optiopay/klar/utils"
 )
 
 const (
@@ -22,14 +24,34 @@ const (
 
 // Image represents Docker image
 type Image struct {
-	Registry string
-	Name     string
-	Tag      string
-	FsLayers []FsLayer
-	Token    string
-	user     string
-	password string
-	client   http.Client
+	Registry      string
+	Name          string
+	Tag           string
+	FsLayers      []FsLayer
+	Token         string
+	user          string
+	password      string
+	client        http.Client
+	digest        string
+	schemaVersion int
+}
+
+func (i *Image) LayerName(index int) string {
+	s := fmt.Sprintf("%s%s", trimDigest(i.digest),
+		trimDigest(i.FsLayers[index].BlobSum))
+	return s
+}
+
+func (i *Image) AnalyzedLayerName() string {
+	index := len(i.FsLayers) - 1
+	if i.schemaVersion == 1 {
+		index = 0
+	}
+	return i.LayerName(index)
+}
+
+func trimDigest(d string) string {
+	return strings.Replace(d, "sha256:", "", 1)
 }
 
 // FsLayer represents a layer in docker image
@@ -39,17 +61,25 @@ type FsLayer struct {
 
 // ImageV1 represents a Manifest V 2, Schema 1 Docker Image
 type imageV1 struct {
-	FsLayers []fsLayer
+	SchemaVersion int
+	FsLayers      []fsLayer
 }
 
 // FsLayer represents a layer in a Manifest V 2, Schema 1 Docker Image
 type fsLayer struct {
 	BlobSum string
 }
 
+type config struct {
+	MediaType string
+	Digest    string
+}
+
 // imageV2 represents Manifest V 2, Schema 2 Docker Image
 type imageV2 struct {
-	Layers []layer
+	SchemaVersion int
+	Config        config
+	Layers        []layer
 }
 
 // Layer represents a layer in a Manifest V 2, Schema 2 Docker Image
@@ -68,7 +98,10 @@ func NewImage(qname, user, password string, insecureTLS, insecureRegistry bool)
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureTLS},
 	}
-	client := http.Client{Transport: tr}
+	client := http.Client{
+		Transport: tr,
+		Timeout:   time.Minute,
+	}
 	registry := dockerHub
 	tag := "latest"
 	var nameParts, tagParts []string
@@ -167,41 +200,38 @@ func (i *Image) Pull() error {
 		}
 	}
 	defer resp.Body.Close()
-	digests, err := extractLayerDigests(resp)
-	if err != nil {
-		return err
-	}
-	i.FsLayers = make([]FsLayer, len(digests))
-	for idx := range digests {
-		i.FsLayers[idx].BlobSum = digests[idx]
-	}
-	return err
+	return parseImageResponse(resp, i)
 }
 
-func extractLayerDigests(resp *http.Response) (digests []string, err error) {
+func parseImageResponse(resp *http.Response, image *Image) error {
 	contentType := resp.Header.Get("Content-Type")
 	if contentType == "application/vnd.docker.distribution.manifest.v2+json" {
 		var imageV2 imageV2
-		if err = json.NewDecoder(resp.Body).Decode(&imageV2); err != nil {
+		if err := json.NewDecoder(resp.Body).Decode(&imageV2); err != nil {
 			fmt.Fprintln(os.Stderr, "Image V2 decode error")
-			return nil, err
+			return err
 		}
-		digests = make([]string, len(imageV2.Layers))
+		image.FsLayers = make([]FsLayer, len(imageV2.Layers))
 		for i := range imageV2.Layers {
-			digests[i] = imageV2.Layers[i].Digest
+			image.FsLayers[i].BlobSum = imageV2.Layers[i].Digest
 		}
+		image.digest = imageV2.Config.Digest
+		image.schemaVersion = imageV2.SchemaVersion
 	} else {
 		var imageV1 imageV1
-		if err = json.NewDecoder(resp.Body).Decode(&imageV1); err != nil {
+		if err := json.NewDecoder(resp.Body).Decode(&imageV1); err != nil {
 			fmt.Fprintln(os.Stderr, "ImageV1 decode error")
-			return nil, err
+			return err
 		}
-		digests = make([]string, len(imageV1.FsLayers))
+		image.FsLayers = make([]FsLayer, len(imageV1.FsLayers))
+		// in schemaVersion 1 layers are in reverse order, so we save them in the same order as v2
+		// base layer is the first
 		for i := range imageV1.FsLayers {
-			digests[i] = imageV1.FsLayers[i].BlobSum
+			image.FsLayers[len(imageV1.FsLayers)-1-i].BlobSum = imageV1.FsLayers[i].BlobSum
 		}
+		image.schemaVersion = imageV1.SchemaVersion
 	}
-	return digests, nil
+	return nil
 }
 
 func (i *Image) requestToken(resp *http.Response) (string, error) {
@@ -267,30 +297,13 @@ func (i *Image) pullReq() (*http.Response, error) {
 	}
 
 	// Prefer manifest schema v2
-	req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json")
-
+	req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json")
+	utils.DumpRequest(req)
 	resp, err := i.client.Do(req)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Get error")
 		return nil, err
 	}
+	utils.DumpResponse(resp)
 	return resp, nil
 }
-
-func dumpRequest(r *http.Request) {
-	dump, err := httputil.DumpRequest(r, true)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Can't dump HTTP request %s\n", err.Error())
-	} else {
-		fmt.Fprintf(os.Stderr, "request_dump: %s\n", string(dump[:]))
-	}
-}
-
-func dumpResponse(r *http.Response) {
-	dump, err := httputil.DumpResponse(r, true)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Can't dump HTTP reqsponse %s\n", err.Error())
-	} else {
-		fmt.Fprintf(os.Stderr, "response_dump: %s\n", string(dump[:]))
-	}
-}

main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/optiopay/klar/clair"
 	"github.com/optiopay/klar/docker"
+	"github.com/optiopay/klar/utils"
 )
 
 type jsonOutput struct {
@@ -25,6 +26,10 @@ func main() {
 		os.Exit(1)
 	}
 
+	if os.Getenv("KLAR_TRACE") != "" {
+		utils.Trace = true
+	}
+
 	clairAddr := os.Getenv("CLAIR_ADDR")
 	if clairAddr == "" {
 		fmt.Fprintf(os.Stderr, "Clair address must be provided\n")

utils/http.go

@@ -0,0 +1,34 @@
+package utils
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	"os"
+)
+
+var Trace bool
+
+func DumpRequest(r *http.Request) {
+	if !Trace {
+		return
+	}
+	dump, err := httputil.DumpRequest(r, true)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Can't dump HTTP request %s\n", err.Error())
+	} else {
+		fmt.Fprintf(os.Stderr, "----> HTTP REQUEST:\n%s\n", string(dump[:]))
+	}
+}
+
+func DumpResponse(r *http.Response) {
+	if !Trace {
+		return
+	}
+	dump, err := httputil.DumpResponse(r, true)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Can't dump HTTP reqsponse %s\n", err.Error())
+	} else {
+		fmt.Fprintf(os.Stderr, "<---- HTTP RESPONSE:\n%s\n", string(dump[:]))
+	}
+}