diff --git a/.travis.yml b/.travis.yml index fdab82cc..79adff03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ install: - go get github.com/golang/lint/golint script: + - go get ./... - go build -v - go vet $(go list ./... | grep -v vendor) - test -z "$(golint ./... | grep -v vendor | tee /dev/stderr)" diff --git a/Dockerfile.dev b/Dockerfile.dev index ba40b965..b30793cd 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,5 @@ FROM golang:alpine RUN apk add --no-cache \ - build-base + build-base \ + git diff --git a/Makefile b/Makefile index b431db29..cbb7d620 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,8 @@ lint: test: @echo "+ $@" + @go get github.com/labstack/echo + @go get github.com/labstack/echo/middleware @go test -v -tags "$(BUILDTAGS) cgo" $(shell go list ./... | grep -v vendor) vet: diff --git a/clair/types.go b/clair/types.go index b24e633a..002bba1e 100644 --- a/clair/types.go +++ b/clair/types.go @@ -56,6 +56,16 @@ type Vulnerability struct { FixedIn []feature `json:"FixedIn,omitempty"` } +// VulnerabilityReport represents the result of a vulnerability scan of a repo. +type VulnerabilityReport struct { + RegistryURL string + Repo string + Tag string + Date string + Vulns []Vulnerability + VulnsBySeverity map[string][]Vulnerability + BadVulns int +} type feature struct { Name string `json:"Name,omitempty"` NamespaceName string `json:"NamespaceName,omitempty"` diff --git a/clair/vulns.go b/clair/vulns.go new file mode 100644 index 00000000..c7001f6b --- /dev/null +++ b/clair/vulns.go @@ -0,0 +1,111 @@ +package clair + +import ( + "fmt" + "strings" + "time" + + "github.com/docker/distribution/manifest/schema1" + "github.com/jessfraz/reg/registry" +) + +// Vulnerabilities scans the given repo and tag +func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema1.SignedManifest) (VulnerabilityReport, error) { + report := VulnerabilityReport{ + RegistryURL: r.Domain, + Repo: repo, + Tag: tag, + Date: time.Now().Local().Format(time.RFC1123), + VulnsBySeverity: make(map[string][]Vulnerability), + } + + // filter out the empty layers + var filteredLayers []schema1.FSLayer + for _, layer := range m.FSLayers { + if layer.BlobSum != EmptyLayerBlobSum { + filteredLayers = append(filteredLayers, layer) + } + } + m.FSLayers = filteredLayers + if len(m.FSLayers) == 0 { + fmt.Printf("No need to analyse image %s:%s as there is no non-emtpy layer", repo, tag) + return report, nil + } + + for i := len(m.FSLayers) - 1; i >= 0; i-- { + // form the clair layer + l, err := c.NewClairLayer(r, repo, m.FSLayers, i) + if err != nil { + return report, err + } + + // post the layer + if _, err := c.PostLayer(l); err != nil { + return report, err + } + } + + vl, err := c.GetLayer(m.FSLayers[0].BlobSum.String(), false, true) + if err != nil { + return report, err + } + + // get the vulns + for _, f := range vl.Features { + for _, v := range f.Vulnerabilities { + report.Vulns = append(report.Vulns, v) + } + } + + vulnsBy := func(sev string, store map[string][]Vulnerability) []Vulnerability { + items, found := store[sev] + if !found { + items = make([]Vulnerability, 0) + store[sev] = items + } + return items + } + + // group by severity + for _, v := range report.Vulns { + sevRow := vulnsBy(v.Severity, report.VulnsBySeverity) + report.VulnsBySeverity[v.Severity] = append(sevRow, v) + } + + // calculate number of bad vulns + report.BadVulns = len(report.VulnsBySeverity["High"]) + len(report.VulnsBySeverity["Critical"]) + len(report.VulnsBySeverity["Defcon1"]) + + return report, nil +} + +// NewClairLayer will form a layer struct required for a clar scan +func (c *Clair) NewClairLayer(r *registry.Registry, image string, fsLayers []schema1.FSLayer, index int) (*Layer, error) { + var parentName string + if index < len(fsLayers)-1 { + parentName = fsLayers[index+1].BlobSum.String() + } + + // form the path + p := strings.Join([]string{r.URL, "v2", image, "blobs", fsLayers[index].BlobSum.String()}, "/") + + // get the token + token, err := r.Token(p) + if err != nil { + return nil, err + } + + h := make(map[string]string) + if token != "" { + h = map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", token), + } + } + + return &Layer{ + Name: fsLayers[index].BlobSum.String(), + Path: p, + ParentName: parentName, + Format: "Docker", + Headers: h, + }, nil +} diff --git a/main.go b/main.go index 72e4804c..f07bab59 100644 --- a/main.go +++ b/main.go @@ -266,6 +266,8 @@ func main() { return err } + // FIXME use clair.Vulnerabilities + // get the manifest m, err := r.ManifestV1(repo, ref) if err != nil { @@ -293,7 +295,7 @@ func main() { for i := len(m.FSLayers) - 1; i >= 0; i-- { // form the clair layer - l, err := utils.NewClairLayer(r, repo, m.FSLayers, i) + l, err := cr.NewClairLayer(r, repo, m.FSLayers, i) if err != nil { return err } diff --git a/server/server.go b/server/server.go index 30d22b4a..c04258b6 100644 --- a/server/server.go +++ b/server/server.go @@ -1,29 +1,22 @@ package main import ( - "encoding/json" "fmt" - "html/template" - "net/http" "os" - "path/filepath" - "strings" "sync" "time" "github.com/Sirupsen/logrus" "github.com/docker/distribution/manifest/schema1" - humanize "github.com/dustin/go-humanize" "github.com/jessfraz/reg/clair" "github.com/jessfraz/reg/registry" "github.com/jessfraz/reg/utils" - wordwrap "github.com/mitchellh/go-wordwrap" "github.com/urfave/cli" ) const ( // VERSION is the binary version. - VERSION = "v0.1.0" + VERSION = "v0.2.0" dockerConfigPath = ".docker/config.json" ) @@ -31,8 +24,8 @@ const ( var ( updating = false wg sync.WaitGroup - tmpl *template.Template r *registry.Registry + cl *clair.Clair ) // preload initializes any global options and configuration @@ -121,127 +114,58 @@ func main() { } } - // get the path to the static directory - wd, err := os.Getwd() - if err != nil { - logrus.Fatal(err) - } - staticDir := filepath.Join(wd, "static") - - // create the template - templateDir := filepath.Join(staticDir, "../templates") - funcMap := template.FuncMap{ - "trim": func(s string) string { - return wordwrap.WrapString(s, 80) - }, - "color": func(s string) string { - switch s = strings.ToLower(s); s { - case "high": - return "danger" - case "critical": - return "danger" - case "defcon1": - return "danger" - case "medium": - return "warning" - case "low": - return "info" - case "negligible": - return "info" - case "unknown": - return "default" - default: - return "default" - } - }, - } - vulns := filepath.Join(templateDir, "vulns.html") - if _, err := os.Stat(vulns); os.IsNotExist(err) { - logrus.Fatalf("Template %s not found", vulns) - } - layout := filepath.Join(templateDir, "layout.html") - if _, err := os.Stat(layout); os.IsNotExist(err) { - logrus.Fatalf("Template %s not found", layout) - } - tmpl = template.Must(template.New("").Funcs(funcMap).ParseFiles(vulns, layout)) - - // create the initial index - logrus.Info("creating initial static index") - if err := createStaticIndex(r, staticDir, "", c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { - logrus.Fatalf("Error creating index: %v", err) - } - // parse the duration dur, err := time.ParseDuration(c.String("interval")) if err != nil { logrus.Fatalf("parsing %s as duration failed: %v", c.String("interval"), err) } + + // create a clair instance if needed + if c.GlobalString("clair") != "" { + cl, err = clair.New(c.GlobalString("clair"), c.GlobalBool("debug")) + if err != nil { + logrus.Warnf("creation of clair failed: %v", err) + } + } ticker := time.NewTicker(dur) go func() { - // create more indexes every X minutes based off interval + // analyse repositories every X minutes based off interval for range ticker.C { if !updating { - logrus.Info("creating timer based static index") - if err := createStaticIndex(r, staticDir, c.GlobalString("clair"), c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { - logrus.Warnf("creating static index failed: %v", err) + logrus.Info("start repository analysis") + start := time.Now() + if err := analyseRepositories(r, cl, c.GlobalBool("debug"), c.GlobalInt("workers")); err != nil { + logrus.Warnf("repository analysis failed: %v", err) wg.Wait() updating = false } wg.Wait() - logrus.Info("finished waiting for vulns wait group") + elapsed := time.Since(start) + logrus.Infof("finished repository analysis in %s", elapsed) } else { - logrus.Warnf("skipping timer based static index update for %s", c.String("interval")) + logrus.Warnf("skipping timer based repository analysis for %s", c.String("interval")) } } }() - // create mux server - mux := http.NewServeMux() - - // static files handler - staticHandler := http.FileServer(http.Dir(staticDir)) - mux.Handle("/", staticHandler) - - // set up the server port := c.String("port") - server := &http.Server{ - Addr: ":" + port, - Handler: mux, - } - logrus.Infof("Starting server on port %q", port) - if c.String("cert") != "" && c.String("key") != "" { - logrus.Fatal(server.ListenAndServeTLS(c.String("cert"), c.String("key"))) - } else { - logrus.Fatal(server.ListenAndServe()) - } + keyfile := c.String("key") + certfile := c.String("cert") + logrus.Fatal(listenAndServe(port, keyfile, certfile, r, cl)) return nil } app.Run(os.Args) } -type data struct { - RegistryURL string - LastUpdated string - Repos []repository -} - -type repository struct { - Name string - Tag string - RepoURI string - CreatedDate string - VulnURI string -} - type v1Compatibility struct { ID string `json:"id"` Created time.Time `json:"created"` } -func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug bool, workers int) error { +func analyseRepositories(r *registry.Registry, cl *clair.Clair, debug bool, workers int) error { updating = true logrus.Info("fetching catalog") repoList, err := r.Catalog("") @@ -250,7 +174,6 @@ func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug b } logrus.Info("fetching tags") - var repos []repository sem := make(chan int, workers) for i, repo := range repoList { // get the tags @@ -260,35 +183,12 @@ func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug b } for j, tag := range tags { // get the manifest - m1, err := r.ManifestV1(repo, tag) if err != nil { logrus.Warnf("getting v1 manifest for %s:%s failed: %v", repo, tag, err) } - var createdDate string - for _, h := range m1.History { - var comp v1Compatibility - if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { - return fmt.Errorf("unmarshal v1compatibility failed: %v", err) - } - createdDate = humanize.Time(comp.Created) - break - } - - repoURI := fmt.Sprintf("%s/%s", r.Domain, repo) - if tag != "latest" { - repoURI += ":" + tag - } - - newrepo := repository{ - Name: repo, - Tag: tag, - RepoURI: repoURI, - CreatedDate: createdDate, - } - - if clairURI != "" { + if cl != nil { wg.Add(1) sem <- 1 go func(repo, tag string, i, j int) { @@ -297,52 +197,21 @@ func createStaticIndex(r *registry.Registry, staticDir, clairURI string, debug b <-sem }() - logrus.Infof("creating vuln static page for %s:%s", repo, tag) + logrus.Infof("search vulnerabilities for %s:%s", repo, tag) - if err := createVulnStaticPage(r, staticDir, clairURI, repo, tag, m1, debug); err != nil { - logrus.Warnf("creating vuln static page for %s:%s failed: %v", repo, tag, err) + if err := searchVulnerabilities(r, cl, repo, tag, m1, debug); err != nil { + logrus.Warnf("searching vulnerabilities for %s:%s failed: %v", repo, tag, err) } }(repo, tag, i, j) - - newrepo.VulnURI = filepath.Join(repo, tag) } - repos = append(repos, newrepo) } } - d := data{ - RegistryURL: r.Domain, - Repos: repos, - LastUpdated: time.Now().Local().Format(time.RFC1123), - } - - logrus.Info("rendering index template") - if err := renderTemplate(staticDir, "index", "index.html", d); err != nil { - return err - } updating = false return nil } -type vulnsReport struct { - RegistryURL string - Repo string - Tag string - Date string - Vulns []clair.Vulnerability - VulnsBySeverity map[string][]clair.Vulnerability - BadVulns int -} - -func createVulnStaticPage(r *registry.Registry, staticDir, clairURI, repo, tag string, m schema1.SignedManifest, debug bool) error { - report := vulnsReport{ - RegistryURL: r.Domain, - Repo: repo, - Tag: tag, - Date: time.Now().Local().Format(time.RFC1123), - VulnsBySeverity: make(map[string][]clair.Vulnerability), - } - +func searchVulnerabilities(r *registry.Registry, cl *clair.Clair, repo, tag string, m schema1.SignedManifest, debug bool) error { // filter out the empty layers var filteredLayers []schema1.FSLayer for _, layer := range m.FSLayers { @@ -356,81 +225,18 @@ func createVulnStaticPage(r *registry.Registry, staticDir, clairURI, repo, tag s return nil } - // initialize clair - cr, err := clair.New(clairURI, debug) - if err != nil { - return err - } - for i := len(m.FSLayers) - 1; i >= 0; i-- { // form the clair layer - l, err := utils.NewClairLayer(r, repo, m.FSLayers, i) + l, err := cl.NewClairLayer(r, repo, m.FSLayers, i) if err != nil { return err } // post the layer - if _, err := cr.PostLayer(l); err != nil { + if _, err := cl.PostLayer(l); err != nil { return err } } - vl, err := cr.GetLayer(m.FSLayers[0].BlobSum.String(), false, true) - if err != nil { - return err - } - - // get the vulns - for _, f := range vl.Features { - for _, v := range f.Vulnerabilities { - report.Vulns = append(report.Vulns, v) - } - } - - vulnsBy := func(sev string, store map[string][]clair.Vulnerability) []clair.Vulnerability { - items, found := store[sev] - if !found { - items = make([]clair.Vulnerability, 0) - store[sev] = items - } - return items - } - - // group by severity - for _, v := range report.Vulns { - sevRow := vulnsBy(v.Severity, report.VulnsBySeverity) - report.VulnsBySeverity[v.Severity] = append(sevRow, v) - } - - // calculate number of bad vulns - report.BadVulns = len(report.VulnsBySeverity["High"]) + len(report.VulnsBySeverity["Critical"]) + len(report.VulnsBySeverity["Defcon1"]) - - path := filepath.Join(repo, tag, "index.html") - if err := renderTemplate(staticDir, "vulns", path, report); err != nil { - return err - } - return nil -} - -func renderTemplate(staticDir, templateName, dest string, data interface{}) error { - // parse & execute the template - logrus.Debugf("executing the template %s", templateName) - - path := filepath.Join(staticDir, dest) - if err := os.MkdirAll(filepath.Dir(path), 0644); err != nil { - return err - } - logrus.Debugf("creating/opening file %s", path) - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - if err := tmpl.ExecuteTemplate(f, templateName, data); err != nil { - f.Close() - return fmt.Errorf("execute template %s failed: %v", templateName, err) - } - return nil } diff --git a/server/static/css/styles.css b/server/static/css/styles.css index 6f696e5e..a392ffa0 100644 --- a/server/static/css/styles.css +++ b/server/static/css/styles.css @@ -194,7 +194,7 @@ td { td:last-child, th:last-child { text-align: right; - padding-right: 0px; + padding-right: 5px; } td a { display: block; @@ -205,6 +205,38 @@ tr.parent a { .parent a:hover { color: #2a2a2a; } + +/*------------------------------------*\ + Loading Indicator +\*------------------------------------*/ +.signal { + border: 2px solid #333; + border-radius: 15px; + height: 15px; + left: 50%; + margin: -8px 0 0 -8px; + opacity: 0; + top: 50%; + width: 15px; + float: right; + animation: pulsate 1s ease-out; + animation-iteration-count: infinite; +} + +@keyframes pulsate { + 0% { + transform: scale(.1); + opacity: 0.0; + } + 50% { + opacity: 1; + } + 100% { + transform: scale(1.2); + opacity: 0; + } +} + /*------------------------------------*\ Footer \*------------------------------------*/ diff --git a/server/static/js/scripts.js b/server/static/js/scripts.js index baf5958c..22bda309 100644 --- a/server/static/js/scripts.js +++ b/server/static/js/scripts.js @@ -23,7 +23,7 @@ function prettyDate(time){ function search(search_val){ var suche = search_val.toLowerCase(); var table = document.getElementById("directory"); - var cellNr = 3; + var cellNr = 1; var ele; for (var r = 1; r < table.rows.length; r++){ ele = table.rows[r].cells[cellNr].innerHTML.replace(/<[^>]+>/g,""); @@ -35,6 +35,25 @@ function search(search_val){ } } +function loadVulnerabilityCount(url){ + var xhr = new XMLHttpRequest(); + xhr.open('GET', url); + xhr.setRequestHeader("Accept-Encoding", "text/json") + xhr.onload = function() { + if (xhr.status === 200) { + var report = JSON.parse(xhr.responseText); + var id = report.Repo + ':' + report.Tag; + var element = document.getElementById(id); + + if (element) { + element.innerHTML = report.BadVulns; + } else { + console.log("element not found for given id ", id); + } + } + }; + xhr.send(); +} var el = document.querySelectorAll('tr:nth-child(2)')[0].querySelectorAll('td:nth-child(2)')[0]; if (el.textContent == 'Parent Directory'){ @@ -72,22 +91,26 @@ our_table.setAttribute('id', 'directory'); var search_input = document.querySelectorAll('input[name="filter"]')[0]; var clear_button = document.querySelectorAll('a.clear')[0]; -if (search_input.value !== ''){ - search(search_input.value); +if (search_input) { + if (search_input.value !== ''){ + search(search_input.value); + } + + search_input.addEventListener('keyup', function(e){ + e.preventDefault(); + search(search_input.value); + }); + + search_input.addEventListener('keypress', function(e){ + if ( e.which == 13 ) { + e.preventDefault(); + } + }); } -search_input.addEventListener('keyup', function(e){ - e.preventDefault(); - search(search_input.value); -}); - -search_input.addEventListener('keypress', function(e){ - if ( e.which == 13 ) { - e.preventDefault(); - } -}); - -clear_button.addEventListener('click', function(e){ - search_input.value = ''; - search(''); -}); +if (clear_button) { + clear_button.addEventListener('click', function(e){ + search_input.value = ''; + search(''); + }); +} diff --git a/server/templates/layout.html b/server/templates/echo/repositories.html similarity index 57% rename from server/templates/layout.html rename to server/templates/echo/repositories.html index d9076235..c0657d4f 100644 --- a/server/templates/layout.html +++ b/server/templates/echo/repositories.html @@ -1,4 +1,4 @@ -{{define "index"}} +{{define "repositories"}} @@ -8,12 +8,12 @@ - {{ .RegistryURL }} + {{ .RegistryDomain }} -

{{ .RegistryURL }}

+

{{ .RegistryDomain }}

clear
@@ -21,32 +21,21 @@
- - - + - {{ range $key, $value := .Repos }} + {{ range $key, $value := .Repositories }} - - + {{ end }} @@ -55,7 +44,6 @@
@jessfraz -

Last Updated: {{ .LastUpdated }}

-{{end}} +{{end}} \ No newline at end of file diff --git a/server/templates/echo/tags.html b/server/templates/echo/tags.html new file mode 100644 index 00000000..6e60250c --- /dev/null +++ b/server/templates/echo/tags.html @@ -0,0 +1,76 @@ +{{define "tags"}} + + + + + + + + + + {{ .RegistryDomain }}/{{ .Name }} + + + + +

{{ .RegistryDomain }}/{{ .Name }}

+
+
NameTagCreatedRepository Name Pull Command
- {{ if $value.VulnURI }}{{ end }} - {{ $value.Name }} - {{ if $value.VulnURI }}{{ end }} - - {{ if $value.VulnURI }}{{ end }} - {{ $value.Tag }} - {{ if $value.VulnURI }}{{ end }} - - {{ if $value.VulnURI }}{{ end }} - {{ $value.CreatedDate }} - {{ if $value.VulnURI }}{{ end }} + + {{ $value.Name }} + - {{ if $value.VulnURI }}{{ end }} - docker pull {{ $value.RepoURI }} - {{ if $value.VulnURI }}{{ end }} + + docker pull {{ $value.URI }} +
+ + + + + + + {{ range $key, $value := .Repositories }} + + + + + + + {{ end }} +
NameTagCreatedVulnerabilities
+ + {{ $value.Name }} + + + + {{ $value.Tag }} + + + {{ $value.Created.Format "02 Jan, 2006 15:04:05 UTC" }} + + +
+ +
+
+ +
+ @jessfraz +
+ + + + + +{{end}} diff --git a/server/templates/vulns.html b/server/templates/echo/vulns.html similarity index 100% rename from server/templates/vulns.html rename to server/templates/echo/vulns.html diff --git a/server/web.go b/server/web.go new file mode 100644 index 00000000..2fb8f97a --- /dev/null +++ b/server/web.go @@ -0,0 +1,337 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "html/template" + "io" + "net/http" + "strings" + + log "github.com/Sirupsen/logrus" + wordwrap "github.com/mitchellh/go-wordwrap" + + "time" + + "net/url" + + "github.com/jessfraz/reg/clair" + "github.com/jessfraz/reg/registry" + "github.com/labstack/echo" + "github.com/labstack/echo/middleware" +) + +type registryController struct { + reg *registry.Registry + cl *clair.Clair +} + +// A Template hold template data +type Template struct { + templates *template.Template +} + +// A Repository holds data after a vulnerability scan of a single repo +type Repository struct { + Name string `json:"name"` + Tag string `json:"tag"` + Created time.Time `json:"created"` + URI string `json:"uri"` + VulnerabilityReport clair.VulnerabilityReport `json:"vulnerability"` +} + +// A AnalysisResult holds all vulnerabilities of a scan +type AnalysisResult struct { + Repositories []Repository `json:"repositories"` + RegistryDomain string `json:"registrydomain"` + Name string `json:"name"` +} + +// Render a template +func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { + return t.templates.ExecuteTemplate(w, name, data) +} + +func listenAndServe(port, keyfile, certfile string, r *registry.Registry, c *clair.Clair) error { + e := echo.New() + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Use(middleware.Static("static")) + + funcMap := template.FuncMap{ + "trim": func(s string) string { + return wordwrap.WrapString(s, 80) + }, + "color": func(s string) string { + switch s = strings.ToLower(s); s { + case "high": + return "danger" + case "critical": + return "danger" + case "defcon1": + return "danger" + case "medium": + return "warning" + case "low": + return "info" + case "negligible": + return "info" + case "unknown": + return "default" + default: + return "default" + } + }, + } + // precompile templates + t := &Template{ + templates: template.Must(template.New("").Funcs(funcMap).ParseGlob("templates/echo/*.html")), + } + e.Renderer = t + + rc := registryController{ + reg: r, + cl: c, + } + + e.GET("/", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, "/repo") + }) + + e.GET("/repo", rc.repositories) + e.GET("/repo/:repo", rc.tags) + e.GET("/repo/:repo/:tag", rc.tag) + e.GET("/repo/:repo/:tag/vulns", rc.vulnerabilities) + + srv := &http.Server{ + Addr: ":" + port, + } + + if keyfile != "" && certfile != "" { + cer, err := tls.LoadX509KeyPair(certfile, keyfile) + if err != nil { + return err + } + srv.TLSConfig = &tls.Config{Certificates: []tls.Certificate{cer}} + } + + return e.StartServer(srv) +} + +func (rc *registryController) repositories(c echo.Context) error { + log.WithFields(log.Fields{ + "method": "repositories", + "context": c, + }).Debug("fetching repositories") + + result := AnalysisResult{} + result.RegistryDomain = rc.reg.Domain + + repoList, err := rc.reg.Catalog("") + if err != nil { + return fmt.Errorf("getting catalog failed: %v", err) + } + for _, repo := range repoList { + log.WithFields(log.Fields{ + "repo": repo, + }).Debug("fetched repo") + repoURI := fmt.Sprintf("%s/%s", rc.reg.Domain, repo) + r := Repository{ + Name: repo, + URI: repoURI, + } + + result.Repositories = append(result.Repositories, r) + } + err = c.Render(http.StatusOK, "repositories", result) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("error during template rendering") + } + return err +} + +// e.GET("/repo/:repo/:tag", rc.tag) +func (rc *registryController) tag(c echo.Context) error { + repo, err := url.QueryUnescape(c.Param("repo")) + if err != nil { + return c.String(http.StatusNotFound, "Given repo can not be unescaped.") + } + if repo == "" { + return c.String(http.StatusNotFound, "No repo given") + } + tag := c.Param("tag") + if tag == "" { + return c.String(http.StatusNotFound, "No tag given") + } + + return c.String(http.StatusOK, fmt.Sprintf("Repo: %s Tag: %s ", repo, tag)) +} + +func (rc *registryController) tags(c echo.Context) error { + repo, err := url.QueryUnescape(c.Param("repo")) + if err != nil { + return c.String(http.StatusNotFound, "Given repo can not be unescaped.") + } + if repo == "" { + return c.String(http.StatusNotFound, "No repo given") + } + log.WithFields(log.Fields{ + "method": "tags", + "context": c, + "repo": repo, + }).Info("fetching tags") + + tags, err := rc.reg.Tags(repo) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + }).Error("getting tags failed.", repo, err) + return c.String(http.StatusNotFound, "No Tags found") + } + + result := AnalysisResult{} + result.RegistryDomain = rc.reg.Domain + result.Name = repo + for _, tag := range tags { + // get the manifest + + m1, err := r.ManifestV1(repo, tag) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn("getting v1 manifest failed") + } + + var createdDate time.Time + for _, h := range m1.History { + var comp v1Compatibility + if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { + msg := "unmarshal v1compatibility failed" + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn(msg) + return c.String(http.StatusInternalServerError, msg) + } + createdDate = comp.Created + break + } + + repoURI := fmt.Sprintf("%s/%s", r.Domain, repo) + if tag != "latest" { + repoURI += ":" + tag + } + r := Repository{ + Name: repo, + Tag: tag, + URI: repoURI, + Created: createdDate, + } + + if rc.cl != nil { + vuln, err := rc.cl.Vulnerabilities(rc.reg, repo, tag, m1) + if err != nil { + msg := "error during vulnerability scanning." + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Error(msg) + return c.String(http.StatusInternalServerError, msg) + } + r.VulnerabilityReport = vuln + } + + result.Repositories = append(result.Repositories, r) + } + err = c.Render(http.StatusOK, "tags", result) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("error during template rendering") + } + return err +} + +func (rc *registryController) vulnerabilities(c echo.Context) error { + repo, err := url.QueryUnescape(c.Param("repo")) + if err != nil { + return c.String(http.StatusNotFound, "Given repo can not be unescaped.") + } + if repo == "" { + return c.String(http.StatusNotFound, "No repo given") + } + tag := c.Param("tag") + if tag == "" { + return c.String(http.StatusNotFound, "No tag given") + } + + m1, err := r.ManifestV1(repo, tag) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn("getting v1 manifest failed") + } + + for _, h := range m1.History { + var comp v1Compatibility + if err := json.Unmarshal([]byte(h.V1Compatibility), &comp); err != nil { + msg := "unmarshal v1compatibility failed" + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Warn(msg) + return c.String(http.StatusInternalServerError, msg) + } + break + } + + result := clair.VulnerabilityReport{} + + if rc.cl != nil { + result, err = rc.cl.Vulnerabilities(rc.reg, repo, tag, m1) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "repo": repo, + "tag": tag, + }).Error("error during vulnerability scanning.") + } + } + + outputType := outputType(c.Request()) + if outputType == "json" { + err = c.JSON(http.StatusOK, result) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("error creating json response") + } + } else { + err = c.Render(http.StatusOK, "vulns", result) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("error during template rendering") + } + } + return err +} + +func outputType(r *http.Request) string { + outputtype := "json" + if r.Header.Get("Accept-Encoding") == "text/html" { + outputtype = "html" + } + return outputtype +} diff --git a/startClair.sh b/startClair.sh new file mode 100755 index 00000000..5ac75fe1 --- /dev/null +++ b/startClair.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + +mkdir -p $PWD/clair_config + +docker rm -f postgres.test || true +docker rm -f clair.test || true + +curl -sfL https://raw.githubusercontent.com/coreos/clair/master/config.example.yaml -o $PWD/clair_config/config.yaml +sed -i -s "s/host=localhost/host=postgres/g" $PWD/clair_config/config.yaml +docker run -d --name postgres.test -e POSTGRES_PASSWORD="" -p 5432:5432 postgres:9.6-alpine +sleep 5 +docker run -it \ + --name clair.test \ + --link postgres.test:postgres \ + -p 6060-6061:6060-6061 \ + -v $PWD/clair_config:/config \ + quay.io/coreos/clair-git:latest -config=/config/config.yaml \ No newline at end of file diff --git a/startRegistryAndCreateImages.sh b/startRegistryAndCreateImages.sh new file mode 100755 index 00000000..9200cfec --- /dev/null +++ b/startRegistryAndCreateImages.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + +docker rm -f registry.test || true +docker run -it -d --name registry.test -p 5000:5000 registry:2.6.0 + +docker pull alpine:3.5 || true + +for repo in `seq 1 20`; +do + for tag in `seq 1 10`; + do + docker tag alpine:3.5 127.0.0.1:5000/company/alpine-${repo}:${tag} + docker push 127.0.0.1:5000/company/alpine-${repo}:${tag} + done +done + +docker logs -f registry.test diff --git a/utils/utils.go b/utils/utils.go index 37ecf26a..3c5d5a87 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -5,11 +5,8 @@ import ( "fmt" "strings" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/docker/api/types" "github.com/docker/docker/cli/config" - "github.com/jessfraz/reg/clair" - "github.com/jessfraz/reg/registry" "github.com/urfave/cli" ) @@ -78,35 +75,3 @@ func GetRepoAndRef(c *cli.Context) (repo, ref string, err error) { return } - -// NewClairLayer creates a clair layer from a docker registry image and fsLayers. -func NewClairLayer(r *registry.Registry, image string, fsLayers []schema1.FSLayer, index int) (*clair.Layer, error) { - var parentName string - if index < len(fsLayers)-1 { - parentName = fsLayers[index+1].BlobSum.String() - } - - // form the path - p := strings.Join([]string{r.URL, "v2", image, "blobs", fsLayers[index].BlobSum.String()}, "/") - - // get the token - token, err := r.Token(p) - if err != nil { - return nil, err - } - - h := make(map[string]string) - if token != "" { - h = map[string]string{ - "Authorization": fmt.Sprintf("Bearer %s", token), - } - } - - return &clair.Layer{ - Name: fsLayers[index].BlobSum.String(), - Path: p, - ParentName: parentName, - Format: "Docker", - Headers: h, - }, nil -}