From fd7fe220e07ab7c414eca2717d68c7d8f290c267 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 6 Apr 2017 12:41:43 +0200 Subject: [PATCH 01/14] First draft of a dynamic frontend, display of vulnerabilities is still not implemented. --- clair/types.go | 10 + clair/vulns.go | 109 ++++++++++ main.go | 2 + server/server.go | 227 ++----------------- server/templates/echo/repositories.html | 59 +++++ server/templates/echo/tags.html | 62 ++++++ server/templates/echo/vulns.html | 78 +++++++ server/web.go | 278 ++++++++++++++++++++++++ startClair.sh | 19 ++ startRegistryAndCreateImages.sh | 19 ++ 10 files changed, 652 insertions(+), 211 deletions(-) create mode 100644 clair/vulns.go create mode 100644 server/templates/echo/repositories.html create mode 100644 server/templates/echo/tags.html create mode 100644 server/templates/echo/vulns.html create mode 100644 server/web.go create mode 100755 startClair.sh create mode 100755 startRegistryAndCreateImages.sh diff --git a/clair/types.go b/clair/types.go index 980ef604..ed009610 100644 --- a/clair/types.go +++ b/clair/types.go @@ -44,6 +44,16 @@ type Vulnerability struct { FixedIn []feature `json:"FixedIn,omitempty"` } +// VulnerabilityReport represents the result of a vulnerability scan of a repo. +type VulnerabilityReport struct { + RegistryURL string + Repo string + Tag string + Date string + Vulns []Vulnerability + VulnsBySeverity map[string][]Vulnerability + BadVulns int +} type feature struct { Name string `json:"Name,omitempty"` NamespaceName string `json:"NamespaceName,omitempty"` diff --git a/clair/vulns.go b/clair/vulns.go new file mode 100644 index 00000000..8cc2ed63 --- /dev/null +++ b/clair/vulns.go @@ -0,0 +1,109 @@ +package clair + +import ( + "fmt" + "strings" + "time" + + "github.com/docker/distribution/manifest/schema1" + "github.com/jessfraz/reg/registry" +) + +func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema1.SignedManifest) (VulnerabilityReport, error) { + report := VulnerabilityReport{ + RegistryURL: r.Domain, + Repo: repo, + Tag: tag, + Date: time.Now().Local().Format(time.RFC1123), + VulnsBySeverity: make(map[string][]Vulnerability), + } + + // filter out the empty layers + var filteredLayers []schema1.FSLayer + for _, layer := range m.FSLayers { + if layer.BlobSum != EmptyLayerBlobSum { + filteredLayers = append(filteredLayers, layer) + } + } + m.FSLayers = filteredLayers + if len(m.FSLayers) == 0 { + fmt.Printf("No need to analyse image %s:%s as there is no non-emtpy layer", repo, tag) + return report, nil + } + + for i := len(m.FSLayers) - 1; i >= 0; i-- { + // form the clair layer + l, err := c.NewClairLayer(r, repo, m.FSLayers, i) + if err != nil { + return report, err + } + + // post the layer + if _, err := c.PostLayer(l); err != nil { + return report, err + } + } + + vl, err := c.GetLayer(m.FSLayers[0].BlobSum.String(), false, true) + if err != nil { + return report, err + } + + // get the vulns + for _, f := range vl.Features { + for _, v := range f.Vulnerabilities { + report.Vulns = append(report.Vulns, v) + } + } + + vulnsBy := func(sev string, store map[string][]Vulnerability) []Vulnerability { + items, found := store[sev] + if !found { + items = make([]Vulnerability, 0) + store[sev] = items + } + return items + } + + // group by severity + for _, v := range report.Vulns { + sevRow := vulnsBy(v.Severity, report.VulnsBySeverity) + report.VulnsBySeverity[v.Severity] = append(sevRow, v) + } + + // calculate number of bad vulns + report.BadVulns = len(report.VulnsBySeverity["High"]) + len(report.VulnsBySeverity["Critical"]) + len(report.VulnsBySeverity["Defcon1"]) + + return report, nil +} + +func (c *Clair) NewClairLayer(r *registry.Registry, image string, fsLayers []schema1.FSLayer, index int) (*Layer, error) { + var parentName string + if index < len(fsLayers)-1 { + parentName = fsLayers[index+1].BlobSum.String() + } + + // form the path + p := strings.Join([]string{r.URL, "v2", image, "blobs", fsLayers[index].BlobSum.String()}, "/") + + // get the token + token, err := r.Token(p) + if err != nil { + return nil, err + } + + h := make(map[string]string) + if token != "" { + h = map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", token), + } + } + + return &Layer{ + Name: fsLayers[index].BlobSum.String(), + Path: p, + ParentName: parentName, + Format: "Docker", + Headers: h, + }, nil +} diff --git a/main.go b/main.go index 97d7e1c9..7b6b9205 100644 --- a/main.go +++ b/main.go @@ -266,6 +266,8 @@ func main() { return err } + // FIXME use clair.Vulnerabilities + // get the manifest m, err := r.ManifestV1(repo, ref) if err != nil { diff --git a/server/server.go b/server/server.go index 30d22b4a..f51a68ca 100644 --- a/server/server.go +++ b/server/server.go @@ -1,23 +1,16 @@ package main import ( - "encoding/json" "fmt" - "html/template" - "net/http" "os" - "path/filepath" - "strings" "sync" "time" "github.com/Sirupsen/logrus" "github.com/docker/distribution/manifest/schema1" - humanize "github.com/dustin/go-humanize" "github.com/jessfraz/reg/clair" "github.com/jessfraz/reg/registry" "github.com/jessfraz/reg/utils" - wordwrap "github.com/mitchellh/go-wordwrap" "github.com/urfave/cli" ) @@ -31,7 +24,6 @@ const ( var ( updating = false wg sync.WaitGroup - tmpl *template.Template r *registry.Registry ) @@ -121,56 +113,6 @@ func main() { } } - // get the path to the static directory - wd, err := os.Getwd() - if err != nil { - logrus.Fatal(err) - } - staticDir := filepath.Join(wd, "static") - - // create the template - templateDir := filepath.Join(staticDir, "../templates") - funcMap := template.FuncMap{ - "trim": func(s string) string { - return wordwrap.WrapString(s, 80) - }, - "color": func(s string) string { - switch s = strings.ToLower(s); s { - case "high": - return "danger" - case "critical": - return "danger" - case "defcon1": - return "danger" - case "medium": - return "warning" - case "low": - return "info" - case "negligible": - return "info" - case "unknown": - return "default" - default: - return "default" - } - }, - } - vulns := filepath.Join(templateDir, "vulns.html") - if _, err := os.Stat(vulns); os.IsNotExist(err) { - logrus.Fatalf("Template %s not found", vulns) - } - layout := filepath.Join(templateDir, "layout.html") - if _, err := os.Stat(layout); os.IsNotExist(err) { - logrus.Fatalf("Template %s not found", layout) - } - tmpl = template.Must(template.New("").Funcs(funcMap).ParseFiles(vulns, layout)) - - // create the initial index - logrus.Info("creating initial static index") - if err := createStaticIndex(r, staticDir, "", c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { - logrus.Fatalf("Error creating index: %v", err) - } - // parse the duration dur, err := time.ParseDuration(c.String("interval")) if err != nil { @@ -179,69 +121,43 @@ func main() { ticker := time.NewTicker(dur) go func() { - // create more indexes every X minutes based off interval + // analyse repositories every X minutes based off interval for range ticker.C { if !updating { - logrus.Info("creating timer based static index") - if err := createStaticIndex(r, staticDir, c.GlobalString("clair"), c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { - logrus.Warnf("creating static index failed: %v", err) + logrus.Info("start repository analysis") + if err := analyseRepositories(r, c.GlobalString("clair"), c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { + logrus.Warnf("repository analysis failed: %v", err) wg.Wait() updating = false } wg.Wait() logrus.Info("finished waiting for vulns wait group") } else { - logrus.Warnf("skipping timer based static index update for %s", c.String("interval")) + logrus.Warnf("skipping timer based repository analysis for %s", c.String("interval")) } } }() - // create mux server - mux := http.NewServeMux() - - // static files handler - staticHandler := http.FileServer(http.Dir(staticDir)) - mux.Handle("/", staticHandler) - - // set up the server port := c.String("port") - server := &http.Server{ - Addr: ":" + port, - Handler: mux, + keyfile := c.String("key") + certfile := c.String("cert") + cl, err := clair.New(c.GlobalString("clair"), c.GlobalBool("debug")) + if err != nil { + logrus.Warnf("creation of clair failed: %v", err) } - logrus.Infof("Starting server on port %q", port) - if c.String("cert") != "" && c.String("key") != "" { - logrus.Fatal(server.ListenAndServeTLS(c.String("cert"), c.String("key"))) - } else { - logrus.Fatal(server.ListenAndServe()) - } - + logrus.Fatal(listenAndServe(port, keyfile, certfile, r, cl)) return nil } app.Run(os.Args) } -type data struct { - RegistryURL string - LastUpdated string - Repos []repository -} - -type repository struct { - Name string - Tag string - RepoURI string - CreatedDate string - VulnURI string -} - type v1Compatibility struct { ID string `json:"id"` Created time.Time `json:"created"` } -func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug bool, workers int) error { +func analyseRepositories(r *registry.Registry, clairURI string, debug bool, workers int) error { updating = true logrus.Info("fetching catalog") repoList, err := r.Catalog("") @@ -250,7 +166,6 @@ func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug b } logrus.Info("fetching tags") - var repos []repository sem := make(chan int, workers) for i, repo := range repoList { // get the tags @@ -266,28 +181,6 @@ func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug b logrus.Warnf("getting v1 manifest for %s:%s failed: %v", repo, tag, err) } - var createdDate string - for _, h := range m1.History { - var comp v1Compatibility - if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { - return fmt.Errorf("unmarshal v1compatibility failed: %v", err) - } - createdDate = humanize.Time(comp.Created) - break - } - - repoURI := fmt.Sprintf("%s/%s", r.Domain, repo) - if tag != "latest" { - repoURI += ":" + tag - } - - newrepo := repository{ - Name: repo, - Tag: tag, - RepoURI: repoURI, - CreatedDate: createdDate, - } - if clairURI != "" { wg.Add(1) sem <- 1 @@ -297,52 +190,21 @@ func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug b <-sem }() - logrus.Infof("creating vuln static page for %s:%s", repo, tag) + logrus.Infof("search vulnerabilities for %s:%s", repo, tag) - if err := createVulnStaticPage(r, staticDir, clairURI, repo, tag, m1, debug); err != nil { - logrus.Warnf("creating vuln static page for %s:%s failed: %v", repo, tag, err) + if err := searchVulnerabilities(r, clairURI, repo, tag, m1, debug); err != nil { + logrus.Warnf("searching vulnerabilities for %s:%s failed: %v", repo, tag, err) } }(repo, tag, i, j) - - newrepo.VulnURI = filepath.Join(repo, tag) } - repos = append(repos, newrepo) } } - d := data{ - RegistryURL: r.Domain, - Repos: repos, - LastUpdated: time.Now().Local().Format(time.RFC1123), - } - - logrus.Info("rendering index template") - if err := renderTemplate(staticDir, "index", "index.html", d); err != nil { - return err - } updating = false return nil } -type vulnsReport struct { - RegistryURL string - Repo string - Tag string - Date string - Vulns []clair.Vulnerability - VulnsBySeverity map[string][]clair.Vulnerability - BadVulns int -} - -func createVulnStaticPage(r *registry.Registry, staticDir, clairURI, repo, tag string, m schema1.SignedManifest, debug bool) error { - report := vulnsReport{ - RegistryURL: r.Domain, - Repo: repo, - Tag: tag, - Date: time.Now().Local().Format(time.RFC1123), - VulnsBySeverity: make(map[string][]clair.Vulnerability), - } - +func searchVulnerabilities(r *registry.Registry, clairURI, repo, tag string, m schema1.SignedManifest, debug bool) error { // filter out the empty layers var filteredLayers []schema1.FSLayer for _, layer := range m.FSLayers { @@ -375,62 +237,5 @@ func createVulnStaticPage(r *registry.Registry, staticDir, clairURI, repo, tag s } } - vl, err := cr.GetLayer(m.FSLayers[0].BlobSum.String(), false, true) - if err != nil { - return err - } - - // get the vulns - for _, f := range vl.Features { - for _, v := range f.Vulnerabilities { - report.Vulns = append(report.Vulns, v) - } - } - - vulnsBy := func(sev string, store map[string][]clair.Vulnerability) []clair.Vulnerability { - items, found := store[sev] - if !found { - items = make([]clair.Vulnerability, 0) - store[sev] = items - } - return items - } - - // group by severity - for _, v := range report.Vulns { - sevRow := vulnsBy(v.Severity, report.VulnsBySeverity) - report.VulnsBySeverity[v.Severity] = append(sevRow, v) - } - - // calculate number of bad vulns - report.BadVulns = len(report.VulnsBySeverity["High"]) + len(report.VulnsBySeverity["Critical"]) + len(report.VulnsBySeverity["Defcon1"]) - - path := filepath.Join(repo, tag, "index.html") - if err := renderTemplate(staticDir, "vulns", path, report); err != nil { - return err - } - return nil -} - -func renderTemplate(staticDir, templateName, dest string, data interface{}) error { - // parse & execute the template - logrus.Debugf("executing the template %s", templateName) - - path := filepath.Join(staticDir, dest) - if err := os.MkdirAll(filepath.Dir(path), 0644); err != nil { - return err - } - logrus.Debugf("creating/opening file %s", path) - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - if err := tmpl.ExecuteTemplate(f, templateName, data); err != nil { - f.Close() - return fmt.Errorf("execute template %s failed: %v", templateName, err) - } - return nil } diff --git a/server/templates/echo/repositories.html b/server/templates/echo/repositories.html new file mode 100644 index 00000000..c0657d4f --- /dev/null +++ b/server/templates/echo/repositories.html @@ -0,0 +1,59 @@ +{{define "repositories"}} + + + + + + + + + + {{ .RegistryDomain }} + + + + +

{{ .RegistryDomain }}

+
+ clear +
+ +
+ + + + + + {{ range $key, $value := .Repositories }} + + + + + + {{ end }} +
Repository NamePull Command
+ + {{ $value.Name }} + + + + docker pull {{ $value.URI }} + +
+
+ +
+ @jessfraz +
+ + + + +{{end}} \ No newline at end of file diff --git a/server/templates/echo/tags.html b/server/templates/echo/tags.html new file mode 100644 index 00000000..87ab27ee --- /dev/null +++ b/server/templates/echo/tags.html @@ -0,0 +1,62 @@ +{{define "tags"}} + + + + + + + + + + {{ .RegistryDomain }}/{{ .Name }} + + + + +

{{ .RegistryDomain }}/{{ .Name }}

+
+ + + + + + + + {{ range $key, $value := .Repositories }} + + + + + + + {{ end }} +
NameTagCreatedVulnerabilities
+ + {{ $value.Name }} + + + + {{ $value.Tag }} + + + {{ $value.Created.Format "02 Jan, 2006 15:04:05 UTC" }} + + {{ $value.VulnerabilityReport.BadVulns }} +
+
+ +
+ @jessfraz +
+ + + + +{{end}} \ No newline at end of file diff --git a/server/templates/echo/vulns.html b/server/templates/echo/vulns.html new file mode 100644 index 00000000..fd79489f --- /dev/null +++ b/server/templates/echo/vulns.html @@ -0,0 +1,78 @@ +{{define "vulns"}} + + + + + + + + + + {{ .RegistryURL }}/{{ .Repo }}:{{ .Tag }} Vulnerability Report + + + + +
+
    +
  1. {{ .RegistryURL }}
    2. +
  2. {{ .Repo }}:{{ .Tag }}
    3. +
+ +
+

{{ .RegistryURL }}/{{ .Repo }}:{{ .Tag }} Vulnerability Report

+
+

Generated on: {{.Date}}

+ + {{if gt .BadVulns 5}} + + {{end}} + +

Summary

+ + +

Details

+ {{range $vulns := .VulnsBySeverity}} + {{range $value := $vulns}} +
+
+

{{$value.Name}} + {{$value.Severity}} +

+
+
+ {{$value.Description}} +
+
+ {{$value.Link}} +
+
+ {{end}} + {{end}} + +
+ + + + +{{end}} diff --git a/server/web.go b/server/web.go new file mode 100644 index 00000000..3074e18b --- /dev/null +++ b/server/web.go @@ -0,0 +1,278 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "html/template" + "io" + "net/http" + "strings" + + log "github.com/Sirupsen/logrus" + wordwrap "github.com/mitchellh/go-wordwrap" + + "time" + + "net/url" + + "github.com/jessfraz/reg/clair" + "github.com/jessfraz/reg/registry" + "github.com/labstack/echo" + "github.com/labstack/echo/middleware" +) + +type registryController struct { + reg *registry.Registry + cl *clair.Clair +} + +// A Template hold template data +type Template struct { + templates *template.Template +} + +type Repository struct { + Name string `json:"name"` + Tag string `json:"tag"` + Created time.Time `json:"created"` + URI string `json:"uri"` + VulnerabilityReport clair.VulnerabilityReport `json:"vulnerability"` +} + +type AnalysisResult struct { + Repositories []Repository `json:"repositories"` + RegistryDomain string `json:"registrydomain"` + Name string `json:"name"` +} + +// Render a template +func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { + return t.templates.ExecuteTemplate(w, name, data) +} + +func listenAndServe(port, keyfile, certfile string, r *registry.Registry, c *clair.Clair) error { + e := echo.New() + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Use(middleware.Static("static")) + + funcMap := template.FuncMap{ + "trim": func(s string) string { + return wordwrap.WrapString(s, 80) + }, + "color": func(s string) string { + switch s = strings.ToLower(s); s { + case "high": + return "danger" + case "critical": + return "danger" + case "defcon1": + return "danger" + case "medium": + return "warning" + case "low": + return "info" + case "negligible": + return "info" + case "unknown": + return "default" + default: + return "default" + } + }, + } + // precompile templates + t := &Template{ + templates: template.Must(template.New("").Funcs(funcMap).ParseGlob("templates/echo/*.html")), + } + e.Renderer = t + + // e.Use(middleware.Logger()) + rc := registryController{ + reg: r, + cl: c, + } + + // e.GET("/repo", nil) + // e.GET("/repo/:repo", nil) + // e.GET("/repo/:repo/:tag", nil) + // e.GET("/repo/:repo/:tag/vulns", nil) + + e.GET("/", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, "/repo") + }) + + e.GET("/repo", rc.repositories) + e.GET("/repo/:repo", rc.tags) + e.GET("/repo/:repo/:tag", rc.tag) + e.GET("/repo/:repo/:tag/vulns", rc.vulnerabilities) + + srv := &http.Server{ + Addr: ":" + port, + } + + if keyfile != "" && certfile != "" { + cer, err := tls.LoadX509KeyPair(certfile, keyfile) + if err != nil { + return err + } + srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{cer}} + } + + return e.StartServer(srv) +} + +func (rc *registryController) repositories(c echo.Context) error { + log.WithFields(log.Fields{ + "method": "repositories", + "context": c, + }).Debug("fetching repositories") + + result := AnalysisResult{} + result.RegistryDomain = rc.reg.Domain + + repoList, err := rc.reg.Catalog("") + if err != nil { + return fmt.Errorf("getting catalog failed: %v", err) + } + for _, repo := range repoList { + log.WithFields(log.Fields{ + "repo": repo, + }).Debug("fetched repo") + repoURI := fmt.Sprintf("%s/%s", rc.reg.Domain, repo) + r := Repository{ + Name: repo, + URI: repoURI, + } + + result.Repositories = append(result.Repositories, r) + } + err = c.Render(http.StatusOK, "repositories", result) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("error during template rendering") + } + return err +} + +// e.GET("/repo/:repo/:tag", rc.tag) +func (rc *registryController) tag(c echo.Context) error { + repo, err := url.QueryUnescape(c.Param("repo")) + if err != nil { + return c.String(http.StatusNotFound, "Given repo can not be unescaped.") + } + if repo == "" { + return c.String(http.StatusNotFound, "No repo given") + } + tag := c.Param("tag") + if tag == "" { + return c.String(http.StatusNotFound, "No tag given") + } + + return c.String(http.StatusOK, fmt.Sprintf("Repo: %s Tag: %s ", repo, tag)) +} + +func (rc *registryController) tags(c echo.Context) error { + repo, err := url.QueryUnescape(c.Param("repo")) + if err != nil { + return c.String(http.StatusNotFound, "Given repo can not be unescaped.") + } + if repo == "" { + return c.String(http.StatusNotFound, "No repo given") + } + log.WithFields(log.Fields{ + "method": "tags", + "context": c, + "repo": repo, + }).Info("fetching tags") + + tags, err := rc.reg.Tags(repo) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + }).Error("getting tags failed.", repo, err) + return c.String(http.StatusNotFound, "No Tags found") + } + + result := AnalysisResult{} + result.RegistryDomain = rc.reg.Domain + result.Name = repo + for _, tag := range tags { + // get the manifest + + m1, err := r.ManifestV1(repo, tag) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn("getting v1 manifest failed") + } + + var createdDate time.Time + for _, h := range m1.History { + var comp v1Compatibility + if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn("unmarshal v1compatibility failed") + return c.String(http.StatusInternalServerError, "unmarshal v1compatibility failed") + } + createdDate = comp.Created + break + } + + repoURI := fmt.Sprintf("%s/%s", r.Domain, repo) + if tag != "latest" { + repoURI += ":" + tag + } + r := Repository{ + Name: repo, + Tag: tag, + URI: repoURI, + Created: createdDate, + } + + if rc.cl != nil { + vuln, err := rc.cl.Vulnerabilities(rc.reg, repo, tag, m1) + // vuln, err := getVulnerabilities(rc.reg, rc.clairUrl, repo, tag, m1, true) // FIXME debug must be configurable + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Error("error during vulnerability scanning.") + } + r.VulnerabilityReport = vuln + } + + result.Repositories = append(result.Repositories, r) + } + err = c.Render(http.StatusOK, "tags", result) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("error during template rendering") + } + return err +} + +func (rc *registryController) vulnerabilities(c echo.Context) error { + repo, err := url.QueryUnescape(c.Param("repo")) + if err != nil { + return c.String(http.StatusNotFound, "Given repo can not be unescaped.") + } + if repo == "" { + return c.String(http.StatusNotFound, "No repo given") + } + tag := c.Param("tag") + if tag == "" { + return c.String(http.StatusNotFound, "No tag given") + } + return nil +} diff --git a/startClair.sh b/startClair.sh new file mode 100755 index 00000000..5ac75fe1 --- /dev/null +++ b/startClair.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + +mkdir -p $PWD/clair_config + +docker rm -f postgres.test || true +docker rm -f clair.test || true + +curl -sfL https://raw.githubusercontent.com/coreos/clair/master/config.example.yaml -o $PWD/clair_config/config.yaml +sed -i -s "s/host=localhost/host=postgres/g" $PWD/clair_config/config.yaml +docker run -d --name postgres.test -e POSTGRES_PASSWORD="" -p 5432:5432 postgres:9.6-alpine +sleep 5 +docker run -it \ + --name clair.test \ + --link postgres.test:postgres \ + -p 6060-6061:6060-6061 \ + -v $PWD/clair_config:/config \ + quay.io/coreos/clair-git:latest -config=/config/config.yaml \ No newline at end of file diff --git a/startRegistryAndCreateImages.sh b/startRegistryAndCreateImages.sh new file mode 100755 index 00000000..d8049f64 --- /dev/null +++ b/startRegistryAndCreateImages.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + +docker rm -f registry.test || true +docker run -it -d --name registry.test -p 5000:5000 registry:2.6.0 + +docker pull alpine:3.5 + +for repo in `seq 1 20`; +do + for tag in `seq 1 10`; + do + docker tag alpine:3.5 127.0.0.1:5000/company/alpine-${repo}:${tag} + docker push 127.0.0.1:5000/company/alpine-${repo}:${tag} + done +done + +docker logs -f registry.test From 369b50e95ccece0fd1968c1573e08b2a18e82980 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 6 Apr 2017 12:57:22 +0200 Subject: [PATCH 02/14] Refactor NewClairLayer to a single place --- main.go | 2 +- server/server.go | 41 +++++++++++++++++++++-------------------- utils/utils.go | 35 ----------------------------------- 3 files changed, 22 insertions(+), 56 deletions(-) diff --git a/main.go b/main.go index 7b6b9205..7265aa09 100644 --- a/main.go +++ b/main.go @@ -295,7 +295,7 @@ func main() { for i := len(m.FSLayers) - 1; i >= 0; i-- { // form the clair layer - l, err := utils.NewClairLayer(r, repo, m.FSLayers, i) + l, err := cr.NewClairLayer(r, repo, m.FSLayers, i) if err != nil { return err } diff --git a/server/server.go b/server/server.go index f51a68ca..c04258b6 100644 --- a/server/server.go +++ b/server/server.go @@ -16,7 +16,7 @@ import ( const ( // VERSION is the binary version. - VERSION = "v0.1.0" + VERSION = "v0.2.0" dockerConfigPath = ".docker/config.json" ) @@ -25,6 +25,7 @@ var ( updating = false wg sync.WaitGroup r *registry.Registry + cl *clair.Clair ) // preload initializes any global options and configuration @@ -118,6 +119,14 @@ func main() { if err != nil { logrus.Fatalf("parsing %s as duration failed: %v", c.String("interval"), err) } + + // create a clair instance if needed + if c.GlobalString("clair") != "" { + cl, err = clair.New(c.GlobalString("clair"), c.GlobalBool("debug")) + if err != nil { + logrus.Warnf("creation of clair failed: %v", err) + } + } ticker := time.NewTicker(dur) go func() { @@ -125,13 +134,15 @@ func main() { for range ticker.C { if !updating { logrus.Info("start repository analysis") - if err := analyseRepositories(r, c.GlobalString("clair"), c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { + start := time.Now() + if err := analyseRepositories(r, cl, c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { logrus.Warnf("repository analysis failed: %v", err) wg.Wait() updating = false } wg.Wait() - logrus.Info("finished waiting for vulns wait group") + elapsed := time.Since(start) + logrus.Infof("finished repository analysis in %s", elapsed) } else { logrus.Warnf("skipping timer based repository analysis for %s", c.String("interval")) } @@ -141,10 +152,7 @@ func main() { port := c.String("port") keyfile := c.String("key") certfile := c.String("cert") - cl, err := clair.New(c.GlobalString("clair"), c.GlobalBool("debug")) - if err != nil { - logrus.Warnf("creation of clair failed: %v", err) - } + logrus.Fatal(listenAndServe(port, keyfile, certfile, r, cl)) return nil } @@ -157,7 +165,7 @@ type v1Compatibility struct { Created time.Time `json:"created"` } -func analyseRepositories(r *registry.Registry, clairURI string, debug bool, workers int) error { +func analyseRepositories(r *registry.Registry, cl *clair.Clair, debug bool, workers int) error { updating = true logrus.Info("fetching catalog") repoList, err := r.Catalog("") @@ -175,13 +183,12 @@ func analyseRepositories(r *registry.Registry, clairURI string, debug bool, work } for j, tag := range tags { // get the manifest - m1, err := r.ManifestV1(repo, tag) if err != nil { logrus.Warnf("getting v1 manifest for %s:%s failed: %v", repo, tag, err) } - if clairURI != "" { + if cl != nil { wg.Add(1) sem <- 1 go func(repo, tag string, i, j int) { @@ -192,7 +199,7 @@ func analyseRepositories(r *registry.Registry, clairURI string, debug bool, work logrus.Infof("search vulnerabilities for %s:%s", repo, tag) - if err := searchVulnerabilities(r, clairURI, repo, tag, m1, debug); err != nil { + if err := searchVulnerabilities(r, cl, repo, tag, m1, debug); err != nil { logrus.Warnf("searching vulnerabilities for %s:%s failed: %v", repo, tag, err) } }(repo, tag, i, j) @@ -204,7 +211,7 @@ func analyseRepositories(r *registry.Registry, clairURI string, debug bool, work return nil } -func searchVulnerabilities(r *registry.Registry, clairURI, repo, tag string, m schema1.SignedManifest, debug bool) error { +func searchVulnerabilities(r *registry.Registry, cl *clair.Clair, repo, tag string, m schema1.SignedManifest, debug bool) error { // filter out the empty layers var filteredLayers []schema1.FSLayer for _, layer := range m.FSLayers { @@ -218,21 +225,15 @@ func searchVulnerabilities(r *registry.Registry, clairURI, repo, tag string, m s return nil } - // initialize clair - cr, err := clair.New(clairURI, debug) - if err != nil { - return err - } - for i := len(m.FSLayers) - 1; i >= 0; i-- { // form the clair layer - l, err := utils.NewClairLayer(r, repo, m.FSLayers, i) + l, err := cl.NewClairLayer(r, repo, m.FSLayers, i) if err != nil { return err } // post the layer - if _, err := cr.PostLayer(l); err != nil { + if _, err := cl.PostLayer(l); err != nil { return err } } diff --git a/utils/utils.go b/utils/utils.go index 37ecf26a..3c5d5a87 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -5,11 +5,8 @@ import ( "fmt" "strings" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/docker/api/types" "github.com/docker/docker/cli/config" - "github.com/jessfraz/reg/clair" - "github.com/jessfraz/reg/registry" "github.com/urfave/cli" ) @@ -78,35 +75,3 @@ func GetRepoAndRef(c *cli.Context) (repo, ref string, err error) { return } - -// NewClairLayer creates a clair layer from a docker registry image and fsLayers. -func NewClairLayer(r *registry.Registry, image string, fsLayers []schema1.FSLayer, index int) (*clair.Layer, error) { - var parentName string - if index < len(fsLayers)-1 { - parentName = fsLayers[index+1].BlobSum.String() - } - - // form the path - p := strings.Join([]string{r.URL, "v2", image, "blobs", fsLayers[index].BlobSum.String()}, "/") - - // get the token - token, err := r.Token(p) - if err != nil { - return nil, err - } - - h := make(map[string]string) - if token != "" { - h = map[string]string{ - "Authorization": fmt.Sprintf("Bearer %s", token), - } - } - - return &clair.Layer{ - Name: fsLayers[index].BlobSum.String(), - Path: p, - ParentName: parentName, - Format: "Docker", - Headers: h, - }, nil -} From 5c7c3f1ed72d2d057077c2e32fbdcb5cce773d91 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 6 Apr 2017 13:32:33 +0200 Subject: [PATCH 03/14] Fix searching for new dynamic frontend --- server/static/js/scripts.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/static/js/scripts.js b/server/static/js/scripts.js index baf5958c..bab6c4e0 100644 --- a/server/static/js/scripts.js +++ b/server/static/js/scripts.js @@ -23,7 +23,7 @@ function prettyDate(time){ function search(search_val){ var suche = search_val.toLowerCase(); var table = document.getElementById("directory"); - var cellNr = 3; + var cellNr = 1; var ele; for (var r = 1; r < table.rows.length; r++){ ele = table.rows[r].cells[cellNr].innerHTML.replace(/<[^>]+>/g,""); From eae3f5ee67d7b680504520f92eeb09fd5b31c496 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 6 Apr 2017 13:33:06 +0200 Subject: [PATCH 04/14] Show vulnerabilities --- server/templates/echo/tags.html | 2 ++ server/web.go | 44 ++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/server/templates/echo/tags.html b/server/templates/echo/tags.html index 87ab27ee..b3ee9054 100644 --- a/server/templates/echo/tags.html +++ b/server/templates/echo/tags.html @@ -38,7 +38,9 @@ {{ $value.Created.Format "02 Jan, 2006 15:04:05 UTC" }} + {{ $value.VulnerabilityReport.BadVulns }} + {{ end }} diff --git a/server/web.go b/server/web.go index 3074e18b..57c1c469 100644 --- a/server/web.go +++ b/server/web.go @@ -274,5 +274,47 @@ func (rc *registryController) vulnerabilities(c echo.Context) error { if tag == "" { return c.String(http.StatusNotFound, "No tag given") } - return nil + + m1, err := r.ManifestV1(repo, tag) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn("getting v1 manifest failed") + } + + for _, h := range m1.History { + var comp v1Compatibility + if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn("unmarshal v1compatibility failed") + return c.String(http.StatusInternalServerError, "unmarshal v1compatibility failed") + } + break + } + + result := clair.VulnerabilityReport{} + + if rc.cl != nil { + result, err = rc.cl.Vulnerabilities(rc.reg, repo, tag, m1) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Error("error during vulnerability scanning.") + } + } + + err = c.Render(http.StatusOK, "vulns", result) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("error during template rendering") + } + return err } From 156d26673624a52b2e5fec0b2ab79a1a56f11526 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 6 Apr 2017 13:34:37 +0200 Subject: [PATCH 05/14] Remove now unneeded templates --- server/templates/layout.html | 71 -------------------------------- server/templates/vulns.html | 78 ------------------------------------ 2 files changed, 149 deletions(-) delete mode 100644 server/templates/layout.html delete mode 100644 server/templates/vulns.html diff --git a/server/templates/layout.html b/server/templates/layout.html deleted file mode 100644 index d9076235..00000000 --- a/server/templates/layout.html +++ /dev/null @@ -1,71 +0,0 @@ -{{define "index"}} - - - - - - - - - - {{ .RegistryURL }} - - - - -

{{ .RegistryURL }}

-
- clear -
- -
- - - - - - - - {{ range $key, $value := .Repos }} - - - - - - - {{ end }} -
NameTagCreatedPull Command
- {{ if $value.VulnURI }}{{ end }} - {{ $value.Name }} - {{ if $value.VulnURI }}{{ end }} - - {{ if $value.VulnURI }}{{ end }} - {{ $value.Tag }} - {{ if $value.VulnURI }}{{ end }} - - {{ if $value.VulnURI }}{{ end }} - {{ $value.CreatedDate }} - {{ if $value.VulnURI }}{{ end }} - - {{ if $value.VulnURI }}{{ end }} - docker pull {{ $value.RepoURI }} - {{ if $value.VulnURI }}{{ end }} -
-
- -
- @jessfraz -

Last Updated: {{ .LastUpdated }}

-
- - - - -{{end}} diff --git a/server/templates/vulns.html b/server/templates/vulns.html deleted file mode 100644 index fd79489f..00000000 --- a/server/templates/vulns.html +++ /dev/null @@ -1,78 +0,0 @@ -{{define "vulns"}} - - - - - - - - - - {{ .RegistryURL }}/{{ .Repo }}:{{ .Tag }} Vulnerability Report - - - - -
-
    -
  1. {{ .RegistryURL }}
    2. -
  2. {{ .Repo }}:{{ .Tag }}
    3. -
- -
-

{{ .RegistryURL }}/{{ .Repo }}:{{ .Tag }} Vulnerability Report

-
-

Generated on: {{.Date}}

- - {{if gt .BadVulns 5}} - - {{end}} - -

Summary

-
    -
  • - {{ len .Vulns }} - Total -
    • - {{range $key, $value := .VulnsBySeverity}} -
  • - {{ len $value }} - {{ $key }} -
    • - {{end}} -
- -

Details

- {{range $vulns := .VulnsBySeverity}} - {{range $value := $vulns}} -
-
-

{{$value.Name}} - {{$value.Severity}} -

-
-
- {{$value.Description}} -
-
- {{$value.Link}} -
-
- {{end}} - {{end}} - -
- - - - -{{end}} From c0bb9b026cc8aed8f360567783d61851fbb46873 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Thu, 6 Apr 2017 14:45:04 +0200 Subject: [PATCH 06/14] cleanup and reusage of error messages --- server/web.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/server/web.go b/server/web.go index 57c1c469..1262d4fc 100644 --- a/server/web.go +++ b/server/web.go @@ -88,17 +88,11 @@ func listenAndServe(port, keyfile, certfile string, r *registry.Registry, c *cla } e.Renderer = t - // e.Use(middleware.Logger()) rc := registryController{ reg: r, cl: c, } - // e.GET("/repo", nil) - // e.GET("/repo/:repo", nil) - // e.GET("/repo/:repo/:tag", nil) - // e.GET("/repo/:repo/:tag/vulns", nil) - e.GET("/", func(c echo.Context) error { return c.Redirect(http.StatusMovedPermanently, "/repo") }) @@ -216,12 +210,13 @@ func (rc *registryController) tags(c echo.Context) error { for _, h := range m1.History { var comp v1Compatibility if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { + msg := "unmarshal v1compatibility failed" log.WithFields(log.Fields{ "error": err, "repo": repo, "tag": tag, - }).Warn("unmarshal v1compatibility failed") - return c.String(http.StatusInternalServerError, "unmarshal v1compatibility failed") + }).Warn(msg) + return c.String(http.StatusInternalServerError, msg) } createdDate = comp.Created break @@ -240,13 +235,14 @@ func (rc *registryController) tags(c echo.Context) error { if rc.cl != nil { vuln, err := rc.cl.Vulnerabilities(rc.reg, repo, tag, m1) - // vuln, err := getVulnerabilities(rc.reg, rc.clairUrl, repo, tag, m1, true) // FIXME debug must be configurable if err != nil { + msg := "error during vulnerability scanning." log.WithFields(log.Fields{ "error": err, "repo": repo, "tag": tag, - }).Error("error during vulnerability scanning.") + }).Error(msg) + return c.String(http.StatusInternalServerError, msg) } r.VulnerabilityReport = vuln } @@ -287,12 +283,13 @@ func (rc *registryController) vulnerabilities(c echo.Context) error { for _, h := range m1.History { var comp v1Compatibility if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { + msg := "unmarshal v1compatibility failed" log.WithFields(log.Fields{ "error": err, "repo": repo, "tag": tag, - }).Warn("unmarshal v1compatibility failed") - return c.String(http.StatusInternalServerError, "unmarshal v1compatibility failed") + }).Warn(msg) + return c.String(http.StatusInternalServerError, msg) } break } From 860b60c624704495d53e3afdcbd5583ac725d29a Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Fri, 7 Apr 2017 11:00:56 +0200 Subject: [PATCH 07/14] Always point to the vulnerabilities --- server/templates/echo/tags.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/templates/echo/tags.html b/server/templates/echo/tags.html index b3ee9054..90249737 100644 --- a/server/templates/echo/tags.html +++ b/server/templates/echo/tags.html @@ -25,12 +25,12 @@ {{ range $key, $value := .Repositories }} - + {{ $value.Name }} - + {{ $value.Tag }} From 4e340ff25ee8405164841744155a5079c0f949eb Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Fri, 7 Apr 2017 19:19:05 +0200 Subject: [PATCH 08/14] Vendor all dependencies --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 800b9671..9697abc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ install: - go get github.com/golang/lint/golint script: + - go get ./... - go build -v - go vet $(go list ./... | grep -v vendor) - test -z "$(golint ./... | grep -v vendor | tee /dev/stderr)" From 616d466f64392ec01bb0a4058cf45d0870cfc05d Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Fri, 7 Apr 2017 19:26:53 +0200 Subject: [PATCH 09/14] Satisfy golint --- clair/vulns.go | 2 ++ server/web.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/clair/vulns.go b/clair/vulns.go index 8cc2ed63..c7001f6b 100644 --- a/clair/vulns.go +++ b/clair/vulns.go @@ -9,6 +9,7 @@ import ( "github.com/jessfraz/reg/registry" ) +// Vulnerabilities scans the given repo and tag func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema1.SignedManifest) (VulnerabilityReport, error) { report := VulnerabilityReport{ RegistryURL: r.Domain, @@ -77,6 +78,7 @@ func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema return report, nil } +// NewClairLayer will form a layer struct required for a clar scan func (c *Clair) NewClairLayer(r *registry.Registry, image string, fsLayers []schema1.FSLayer, index int) (*Layer, error) { var parentName string if index < len(fsLayers)-1 { diff --git a/server/web.go b/server/web.go index 1262d4fc..e9398f66 100644 --- a/server/web.go +++ b/server/web.go @@ -32,6 +32,7 @@ type Template struct { templates *template.Template } +// A Repository holds data after a vulnerability scan of a single repo type Repository struct { Name string `json:"name"` Tag string `json:"tag"` @@ -40,6 +41,7 @@ type Repository struct { VulnerabilityReport clair.VulnerabilityReport `json:"vulnerability"` } +// A AnalysisResult holds all vulnerabilities of a scan type AnalysisResult struct { Repositories []Repository `json:"repositories"` RegistryDomain string `json:"registrydomain"` From 02503f91d930e438c869585a1505f0486379cf68 Mon Sep 17 00:00:00 2001 From: Markus Fensterer Date: Wed, 19 Apr 2017 15:17:33 +0200 Subject: [PATCH 10/14] load BadVulns count with AJAX calls --- server/static/css/styles.css | 34 +++++++++++++++++++- server/static/js/scripts.js | 56 +++++++++++++++++++++++---------- server/templates/echo/tags.html | 20 +++++++++--- server/web.go | 53 +++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 22 deletions(-) diff --git a/server/static/css/styles.css b/server/static/css/styles.css index 6f696e5e..a392ffa0 100644 --- a/server/static/css/styles.css +++ b/server/static/css/styles.css @@ -194,7 +194,7 @@ td { td:last-child, th:last-child { text-align: right; - padding-right: 0px; + padding-right: 5px; } td a { display: block; @@ -205,6 +205,38 @@ tr.parent a { .parent a:hover { color: #2a2a2a; } + +/*------------------------------------*\ + Loading Indicator +\*------------------------------------*/ +.signal { + border: 2px solid #333; + border-radius: 15px; + height: 15px; + left: 50%; + margin: -8px 0 0 -8px; + opacity: 0; + top: 50%; + width: 15px; + float: right; + animation: pulsate 1s ease-out; + animation-iteration-count: infinite; +} + +@keyframes pulsate { + 0% { + transform: scale(.1); + opacity: 0.0; + } + 50% { + opacity: 1; + } + 100% { + transform: scale(1.2); + opacity: 0; + } +} + /*------------------------------------*\ Footer \*------------------------------------*/ diff --git a/server/static/js/scripts.js b/server/static/js/scripts.js index bab6c4e0..49a68e0b 100644 --- a/server/static/js/scripts.js +++ b/server/static/js/scripts.js @@ -35,6 +35,24 @@ function search(search_val){ } } +function loadVulnerabilityCount(url){ + var xhr = new XMLHttpRequest(); + xhr.open('GET', url); + xhr.onload = function() { + if (xhr.status === 200) { + var report = JSON.parse(xhr.responseText); + var id = report.Repo + ':' + report.Tag; + var element = document.getElementById(id); + + if (element) { + element.innerHTML = report.BadVulns; + } else { + console.log("element not found for given id ", id); + } + } + }; + xhr.send(); +} var el = document.querySelectorAll('tr:nth-child(2)')[0].querySelectorAll('td:nth-child(2)')[0]; if (el.textContent == 'Parent Directory'){ @@ -72,22 +90,26 @@ our_table.setAttribute('id', 'directory'); var search_input = document.querySelectorAll('input[name="filter"]')[0]; var clear_button = document.querySelectorAll('a.clear')[0]; -if (search_input.value !== ''){ - search(search_input.value); +if (search_input) { + if (search_input.value !== ''){ + search(search_input.value); + } + + search_input.addEventListener('keyup', function(e){ + e.preventDefault(); + search(search_input.value); + }); + + search_input.addEventListener('keypress', function(e){ + if ( e.which == 13 ) { + e.preventDefault(); + } + }); } -search_input.addEventListener('keyup', function(e){ - e.preventDefault(); - search(search_input.value); -}); - -search_input.addEventListener('keypress', function(e){ - if ( e.which == 13 ) { - e.preventDefault(); - } -}); - -clear_button.addEventListener('click', function(e){ - search_input.value = ''; - search(''); -}); +if (clear_button) { + clear_button.addEventListener('click', function(e){ + search_input.value = ''; + search(''); + }); +} diff --git a/server/templates/echo/tags.html b/server/templates/echo/tags.html index 90249737..255354a0 100644 --- a/server/templates/echo/tags.html +++ b/server/templates/echo/tags.html @@ -37,9 +37,9 @@ {{ $value.Created.Format "02 Jan, 2006 15:04:05 UTC" }} - - - {{ $value.VulnerabilityReport.BadVulns }} + + +
@@ -51,6 +51,18 @@ @jessfraz + -{{end}} \ No newline at end of file +{{end}} diff --git a/server/web.go b/server/web.go index e9398f66..66f90b2e 100644 --- a/server/web.go +++ b/server/web.go @@ -103,6 +103,7 @@ func listenAndServe(port, keyfile, certfile string, r *registry.Registry, c *cla e.GET("/repo/:repo", rc.tags) e.GET("/repo/:repo/:tag", rc.tag) e.GET("/repo/:repo/:tag/vulns", rc.vulnerabilities) + e.GET("/repo/:repo/:tag/report", rc.report) srv := &http.Server{ Addr: ":" + port, @@ -260,6 +261,58 @@ func (rc *registryController) tags(c echo.Context) error { return err } +func (rc *registryController) report(c echo.Context) error { + repo, err := url.QueryUnescape(c.Param("repo")) + if err != nil { + return c.String(http.StatusNotFound, "Given repo can not be unescaped.") + } + if repo == "" { + return c.String(http.StatusNotFound, "No repo given") + } + tag := c.Param("tag") + if tag == "" { + return c.String(http.StatusNotFound, "No tag given") + } + + m1, err := r.ManifestV1(repo, tag) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn("getting v1 manifest failed") + } + + for _, h := range m1.History { + var comp v1Compatibility + if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { + msg := "unmarshal v1compatibility failed" + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn(msg) + return c.String(http.StatusInternalServerError, msg) + } + break + } + + result := clair.VulnerabilityReport{} + + if rc.cl != nil { + result, err = rc.cl.Vulnerabilities(rc.reg, repo, tag, m1) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Error("error during vulnerability scanning.") + } + } + + return c.JSON(http.StatusOK, result) +} + func (rc *registryController) vulnerabilities(c echo.Context) error { repo, err := url.QueryUnescape(c.Param("repo")) if err != nil { From df8b759dc81ab4316d22309db71a26bfa1ba6293 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Mon, 24 Apr 2017 16:14:55 +0200 Subject: [PATCH 11/14] Make sample shell script more robust --- startRegistryAndCreateImages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startRegistryAndCreateImages.sh b/startRegistryAndCreateImages.sh index d8049f64..9200cfec 100755 --- a/startRegistryAndCreateImages.sh +++ b/startRegistryAndCreateImages.sh @@ -5,7 +5,7 @@ set -e docker rm -f registry.test || true docker run -it -d --name registry.test -p 5000:5000 registry:2.6.0 -docker pull alpine:3.5 +docker pull alpine:3.5 || true for repo in `seq 1 20`; do From c56cc83e2a14919f354caecbdf501708f3c32c04 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Mon, 24 Apr 2017 16:25:32 +0200 Subject: [PATCH 12/14] Make tests work again --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b431db29..cbb7d620 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,8 @@ lint: test: @echo "+ $@" + @go get github.com/labstack/echo + @go get github.com/labstack/echo/middleware @go test -v -tags "$(BUILDTAGS) cgo" $(shell go list ./... | grep -v vendor) vet: From 3d0dd20f9c900c7beec404989c987cc609fa0da7 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Mon, 24 Apr 2017 17:05:46 +0200 Subject: [PATCH 13/14] Add git to be able to do go get --- Dockerfile.dev | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index ba40b965..b30793cd 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,5 @@ FROM golang:alpine RUN apk add --no-cache \ - build-base + build-base \ + git From b8cc50b46a1830ffbd6f280a17150a2e4e271b3a Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Mon, 24 Apr 2017 18:20:58 +0200 Subject: [PATCH 14/14] Remove code duplicates --- server/static/js/scripts.js | 1 + server/templates/echo/tags.html | 2 +- server/web.go | 81 ++++++++++----------------------- 3 files changed, 25 insertions(+), 59 deletions(-) diff --git a/server/static/js/scripts.js b/server/static/js/scripts.js index 49a68e0b..22bda309 100644 --- a/server/static/js/scripts.js +++ b/server/static/js/scripts.js @@ -38,6 +38,7 @@ function search(search_val){ function loadVulnerabilityCount(url){ var xhr = new XMLHttpRequest(); xhr.open('GET', url); + xhr.setRequestHeader("Accept-Encoding", "text/json") xhr.onload = function() { if (xhr.status === 200) { var report = JSON.parse(xhr.responseText); diff --git a/server/templates/echo/tags.html b/server/templates/echo/tags.html index 255354a0..6e60250c 100644 --- a/server/templates/echo/tags.html +++ b/server/templates/echo/tags.html @@ -54,7 +54,7 @@