refactor client

Signed-off-by: Jess Frazelle <acidburn@microsoft.com>
This commit is contained in:
Jess Frazelle 2018-03-06 09:12:29 -05:00
parent 5088b3b142
commit a3b459b1a5
No known key found for this signature in database
GPG key ID: 18F3685C0022BFF3
15 changed files with 397 additions and 381 deletions

View file

@ -11,7 +11,7 @@ import (
)
// Vulnerabilities scans the given repo and tag
func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema1.SignedManifest) (VulnerabilityReport, error) {
func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string) (VulnerabilityReport, error) {
report := VulnerabilityReport{
RegistryURL: r.Domain,
Repo: repo,
@ -20,13 +20,20 @@ func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema
VulnsBySeverity: make(map[string][]Vulnerability),
}
// filter out the empty layers
// Get the v1 manifest to pass to clair.
m, err := r.ManifestV1(repo, tag)
if err != nil {
return report, fmt.Errorf("getting the v1 manifest for %s:%s failed: %v", repo, tag, err)
}
// Filter out the empty layers.
var filteredLayers []schema1.FSLayer
for _, layer := range m.FSLayers {
if layer.BlobSum != EmptyLayerBlobSum {
filteredLayers = append(filteredLayers, layer)
}
}
m.FSLayers = filteredLayers
if len(m.FSLayers) == 0 {
fmt.Printf("No need to analyse image %s:%s as there is no non-emtpy layer", repo, tag)
@ -34,13 +41,13 @@ func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema
}
for i := len(m.FSLayers) - 1; i >= 0; i-- {
// form the clair layer
// Form the clair layer.
l, err := c.NewClairLayer(r, repo, m.FSLayers, i)
if err != nil {
return report, err
}
// post the layer
// Post the layer.
if _, err := c.PostLayer(l); err != nil {
return report, err
}
@ -51,7 +58,7 @@ func (c *Clair) Vulnerabilities(r *registry.Registry, repo, tag string, m schema
return report, err
}
// get the vulns
// Get the vulns.
for _, f := range vl.Features {
for _, v := range f.Vulnerabilities {
report.Vulns = append(report.Vulns, v)

31
delete.go Normal file
View file

@ -0,0 +1,31 @@
package main
import (
"fmt"
"github.com/jessfraz/reg/repoutils"
"github.com/urfave/cli"
)
var deleteCommand = cli.Command{
Name: "delete",
Aliases: []string{"rm"},
Usage: "delete a specific reference of a repository",
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
repo, ref, err := repoutils.GetRepoAndRef(c.Args()[0])
if err != nil {
return err
}
if err := r.Delete(repo, ref); err != nil {
return fmt.Errorf("Delete %s@%s failed: %v", repo, ref, err)
}
fmt.Printf("Deleted %s@%s\n", repo, ref)
return nil
},
}

52
layer.go Normal file
View file

@ -0,0 +1,52 @@
package main
import (
"fmt"
"io/ioutil"
"os"
"github.com/jessfraz/reg/repoutils"
digest "github.com/opencontainers/go-digest"
"github.com/urfave/cli"
)
var layerCommand = cli.Command{
Name: "layer",
Aliases: []string{"download"},
Usage: "download a layer for the specific reference of a repository",
Flags: []cli.Flag{
cli.StringFlag{
Name: "output, o",
Usage: "output file, default to stdout",
},
},
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
repo, ref, err := repoutils.GetRepoAndRef(c.Args()[0])
if err != nil {
return err
}
layer, err := r.DownloadLayer(repo, digest.FromString(ref))
if err != nil {
return err
}
defer layer.Close()
b, err := ioutil.ReadAll(layer)
if err != nil {
return err
}
if c.String("output") != "" {
return ioutil.WriteFile(c.String("output"), b, 0644)
}
fmt.Fprint(os.Stdout, string(b))
return nil
},
}

62
list.go Normal file
View file

@ -0,0 +1,62 @@
package main
import (
"fmt"
"os"
"strings"
"sync"
"text/tabwriter"
"github.com/urfave/cli"
)
var listCommand = cli.Command{
Name: "list",
Aliases: []string{"ls"},
Usage: "list all repositories",
Action: func(c *cli.Context) error {
// Get the repositories via catalog.
repos, err := r.Catalog("")
if err != nil {
return err
}
fmt.Printf("Repositories for %s\n", auth.ServerAddress)
// Setup the tab writer.
w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
// Print header.
fmt.Fprintln(w, "REPO\tTAGS")
var (
l sync.Mutex
wg sync.WaitGroup
)
wg.Add(len(repos))
for _, repo := range repos {
go func(repo string) {
// Get the tags and print to stdout.
tags, err := r.Tags(repo)
if err != nil {
fmt.Printf("Get tags of [%s] error: %s", repo, err)
}
out := fmt.Sprintf("%s\t%s\n", repo, strings.Join(tags, ", "))
// Lock around the tabwriter to prevent garbled output.
// See: https://github.com/jessfraz/reg/issues/54
l.Lock()
w.Write([]byte(out))
l.Unlock()
wg.Done()
}(repo)
}
wg.Wait()
w.Flush()
return nil
},
}

17
list_test.go Normal file
View file

@ -0,0 +1,17 @@
package main
import "testing"
func TestList(t *testing.T) {
out, err := run("ls")
if err != nil {
t.Fatalf("output: %s, error: %v", string(out), err)
}
expected := `Repositories for localhost:5000
REPO TAGS
alpine latest
`
if out != expected {
t.Fatalf("expected: %s\ngot: %s", expected, out)
}
}

374
main.go
View file

@ -1,22 +1,14 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"strings"
"sync"
"text/tabwriter"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/docker/api/types"
"github.com/jessfraz/reg/clair"
"github.com/jessfraz/reg/registry"
"github.com/jessfraz/reg/utils"
"github.com/jessfraz/reg/repoutils"
"github.com/jessfraz/reg/version"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
@ -30,43 +22,6 @@ var (
r *registry.Registry
)
// preload initializes any global options and configuration
// before the main or sub commands are run.
func preload(c *cli.Context) (err error) {
if c.GlobalBool("debug") {
logrus.SetLevel(logrus.DebugLevel)
}
if len(c.Args()) > 0 {
if c.Args()[0] != "help" {
auth, err = utils.GetAuthConfig(c.GlobalString("username"), c.GlobalString("password"), c.GlobalString("registry"))
if err != nil {
return err
}
// Prevent non-ssl unless explicitly forced
if !c.GlobalBool("force-non-ssl") && strings.HasPrefix(auth.ServerAddress, "http:") {
return fmt.Errorf("Attempt to use insecure protocol! Use non-ssl option to force")
}
// create the registry client
if c.GlobalBool("insecure") {
r, err = registry.NewInsecure(auth, c.GlobalBool("debug"))
if err != nil {
return err
}
} else {
r, err = registry.New(auth, c.GlobalBool("debug"))
if err != nil {
return err
}
}
}
}
return nil
}
func main() {
app := cli.NewApp()
app.Name = "reg"
@ -74,7 +29,7 @@ func main() {
app.Author = "@jessfraz"
app.Email = "no-reply@butts.com"
app.Usage = "Docker registry v2 client."
app.Before = preload
app.Flags = []cli.Flag{
cli.BoolFlag{
Name: "debug, d",
@ -99,321 +54,54 @@ func main() {
cli.StringFlag{
Name: "registry, r",
Usage: "URL to the private registry (ex. r.j3ss.co)",
Value: utils.DefaultDockerRegistry,
Value: repoutils.DefaultDockerRegistry,
},
}
app.Commands = []cli.Command{
{
Name: "delete",
Aliases: []string{"rm"},
Usage: "delete a specific reference of a repository",
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
deleteCommand,
layerCommand,
listCommand,
manifestCommand,
tagsCommand,
vulnsCommand,
}
repo, ref, err := utils.GetRepoAndRef(c.Args()[0])
app.Before = func(c *cli.Context) (err error) {
// Preload initializes any global options and configuration
// before the main or sub commands are run.
if c.GlobalBool("debug") {
logrus.SetLevel(logrus.DebugLevel)
}
if len(c.Args()) > 0 {
if c.Args()[0] != "help" {
auth, err = repoutils.GetAuthConfig(c.GlobalString("username"), c.GlobalString("password"), c.GlobalString("registry"))
if err != nil {
return err
}
if err := r.Delete(repo, ref); err != nil {
return fmt.Errorf("Delete %s@%s failed: %v", repo, ref, err)
}
fmt.Printf("Deleted %s@%s\n", repo, ref)
return nil
},
},
{
Name: "list",
Aliases: []string{"ls"},
Usage: "list all repositories",
Action: func(c *cli.Context) error {
// get the repositories via catalog
repos, err := r.Catalog("")
if err != nil {
return err
// Prevent non-ssl unless explicitly forced
if !c.GlobalBool("force-non-ssl") && strings.HasPrefix(auth.ServerAddress, "http:") {
return fmt.Errorf("Attempt to use insecure protocol! Use non-ssl option to force")
}
fmt.Printf("Repositories for %s\n", auth.ServerAddress)
// setup the tab writer
w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
// print header
fmt.Fprintln(w, "REPO\tTAGS")
var l sync.Mutex
var wg sync.WaitGroup
wg.Add(len(repos))
for _, repo := range repos {
go func(repo string) {
// get the tags and print to stdout
tags, err := r.Tags(repo)
if err != nil {
fmt.Printf("Get tags of [%s] error: %s", repo, err)
}
out := fmt.Sprintf("%s\t%s\n", repo, strings.Join(tags, ", "))
l.Lock()
w.Write([]byte(out))
l.Unlock()
wg.Done()
}(repo)
}
wg.Wait()
w.Flush()
return nil
},
},
{
Name: "manifest",
Usage: "get the json manifest for the specific reference of a repository",
Flags: []cli.Flag{
cli.BoolFlag{
Name: "v1",
Usage: "force get v1 manifest",
},
},
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
repo, ref, err := utils.GetRepoAndRef(c.Args()[0])
if err != nil {
return err
}
var manifest interface{}
if c.Bool("v1") {
manifest, err = r.ManifestV1(repo, ref)
// create the registry client
if c.GlobalBool("insecure") {
r, err = registry.NewInsecure(auth, c.GlobalBool("debug"))
if err != nil {
return err
}
} else {
manifest, err = r.Manifest(repo, ref)
r, err = registry.New(auth, c.GlobalBool("debug"))
if err != nil {
return err
}
}
}
}
b, err := json.MarshalIndent(manifest, " ", " ")
if err != nil {
return err
}
fmt.Println(string(b))
return nil
},
},
{
Name: "tags",
Usage: "get the tags for a repository",
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
tags, err := r.Tags(c.Args()[0])
if err != nil {
return err
}
// print the tags
fmt.Println(strings.Join(tags, "\n"))
return nil
},
},
{
Name: "download",
Aliases: []string{"layer"},
Usage: "download a layer for the specific reference of a repository",
Flags: []cli.Flag{
cli.StringFlag{
Name: "output, o",
Usage: "output file, default to stdout",
},
},
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
repo, ref, err := utils.GetRepoAndRef(c.Args()[0])
if err != nil {
return err
}
layer, err := r.DownloadLayer(repo, digest.FromString(ref))
if err != nil {
return err
}
defer layer.Close()
b, err := ioutil.ReadAll(layer)
if err != nil {
return err
}
if c.String("output") != "" {
return ioutil.WriteFile(c.String("output"), b, 0644)
}
fmt.Fprint(os.Stdout, string(b))
return nil
},
},
{
Name: "vulns",
Usage: "get a vulnerability report for the image from CoreOS Clair",
Flags: []cli.Flag{
cli.StringFlag{
Name: "clair",
Usage: "url to clair instance",
},
cli.IntFlag{
Name: "fixable-threshold",
Usage: "number of fixable issues permitted",
Value: 0,
},
},
Action: func(c *cli.Context) error {
if c.String("clair") == "" {
return errors.New("clair url cannot be empty, pass --clair")
}
if c.Int("fixable-threshold") < 0 {
return errors.New("fixable threshold must be a positive integer")
}
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
repo, ref, err := utils.GetRepoAndRef(c.Args()[0])
if err != nil {
return err
}
// FIXME use clair.Vulnerabilities
// get the manifest
m, err := r.ManifestV1(repo, ref)
if err != nil {
return err
}
// filter out the empty layers
var filteredLayers []schema1.FSLayer
for i, layer := range m.FSLayers {
if !clair.IsEmptyLayer(layer.BlobSum) {
filteredLayers = append(filteredLayers, layer)
logrus.Debugf("%d: layer=%s <-- append", i, layer)
} else {
logrus.Debugf("%d: layer=%s", i, layer)
}
}
m.FSLayers = filteredLayers
if len(m.FSLayers) == 0 {
fmt.Printf("No need to analyse image %s:%s as there is no non-emtpy layer", repo, ref)
return nil
}
fmt.Printf("Analysing %d layers\n", len(m.FSLayers))
// initialize clair
cr, err := clair.New(c.String("clair"), c.GlobalBool("debug"))
if err != nil {
return err
}
for i := len(m.FSLayers) - 1; i >= 0; i-- {
// form the clair layer
l, err := cr.NewClairLayer(r, repo, m.FSLayers, i)
if err != nil {
return err
}
// post the layer
if _, err := cr.PostLayer(l); err != nil {
return err
}
}
vl, err := cr.GetLayer(m.FSLayers[0].BlobSum.String(), false, true)
if err != nil {
return err
}
// get the vulns
var vulns []clair.Vulnerability
for _, f := range vl.Features {
for _, v := range f.Vulnerabilities {
vulns = append(vulns, v)
}
}
fmt.Printf("Found %d vulnerabilities \n", len(vulns))
vulnsBy := func(sev string, store map[string][]clair.Vulnerability) []clair.Vulnerability {
items, found := store[sev]
if !found {
items = make([]clair.Vulnerability, 0)
store[sev] = items
}
return items
}
// group by severity
store := make(map[string][]clair.Vulnerability)
for _, v := range vulns {
sevRow := vulnsBy(v.Severity, store)
store[v.Severity] = append(sevRow, v)
if len(v.FixedBy) > 0 {
fixRow := vulnsBy("Fixable", store)
store["Fixable"] = append(fixRow, v)
}
}
// iterate over the priorities list
iteratePriorities := func(f func(sev string)) {
for _, sev := range clair.Priorities {
if len(store[sev]) != 0 {
f(sev)
}
}
}
iteratePriorities(func(sev string) {
for _, v := range store[sev] {
if sev == "Fixable" {
fmt.Printf("%s: [%s] \n%s\n%s\n", v.Name, v.Severity+" - Fixable", v.Description, v.Link)
fmt.Printf("Fixed by: %s\n", v.FixedBy)
} else {
fmt.Printf("%s: [%s] \n%s\n%s\n", v.Name, v.Severity, v.Description, v.Link)
}
fmt.Println("-----------------------------------------")
}
})
iteratePriorities(func(sev string) {
fmt.Printf("%s: %d\n", sev, len(store[sev]))
})
// return an error if there are more than 1 fixable vulns
lenFixableVulns := len(store["Fixable"])
if lenFixableVulns > c.Int("fixable-threshold") {
logrus.Fatalf("%d fixable vulnerabilities found", lenFixableVulns)
}
// return an error if there are more than 10 bad vulns
lenBadVulns := len(store["High"]) + len(store["Critical"]) + len(store["Defcon1"])
if lenBadVulns > 10 {
logrus.Fatalf("%d bad vulnerabilities found", lenBadVulns)
}
return nil
},
},
return nil
}
if err := app.Run(os.Args); err != nil {

View file

@ -95,17 +95,3 @@ func run(args ...string) (string, error) {
out, err := cmd.CombinedOutput()
return string(out), err
}
func TestList(t *testing.T) {
out, err := run("ls")
if err != nil {
t.Fatalf("output: %s, error: %v", string(out), err)
}
expected := `Repositories for localhost:5000
REPO TAGS
alpine latest
`
if out != expected {
t.Fatalf("expected: %s\ngot: %s", expected, out)
}
}

54
manifest.go Normal file
View file

@ -0,0 +1,54 @@
package main
import (
"encoding/json"
"fmt"
"github.com/jessfraz/reg/repoutils"
"github.com/urfave/cli"
)
var manifestCommand = cli.Command{
Name: "manifest",
Usage: "get the json manifest for the specific reference of a repository",
Flags: []cli.Flag{
cli.BoolFlag{
Name: "v1",
Usage: "force the version of the manifest retreived to v1",
},
},
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
repo, ref, err := repoutils.GetRepoAndRef(c.Args()[0])
if err != nil {
return err
}
var manifest interface{}
if c.Bool("v1") {
// Get the v1 manifest if it was explicitly asked for.
manifest, err = r.ManifestV1(repo, ref)
if err != nil {
return err
}
} else {
// Get the v2 manifest.
manifest, err = r.Manifest(repo, ref)
if err != nil {
return err
}
}
b, err := json.MarshalIndent(manifest, " ", " ")
if err != nil {
return err
}
fmt.Println(string(b))
return nil
},
}

View file

@ -31,7 +31,7 @@ func (asm authServiceMock) equalTo(v *authService) bool {
return true
}
func Test_parseChallenge(t *testing.T) {
func TestParseChallenge(t *testing.T) {
challengeHeaderCases := []challengeTestCase{
{
header: `Bearer realm="https://foobar.com/api/v1/token",service=foobar.com,scope=""`,
@ -41,6 +41,7 @@ func Test_parseChallenge(t *testing.T) {
},
},
}
for _, tc := range challengeHeaderCases {
val, err := parseChallenge(tc.header)
if err != nil && !strings.Contains(err.Error(), tc.errorString) {

View file

@ -1,4 +1,4 @@
package utils
package repoutils
import (
"errors"

View file

@ -0,0 +1 @@
package repoutils

View file

@ -214,24 +214,10 @@ func (rc *registryController) vulnerabilitiesHandler(w http.ResponseWriter, r *h
return
}
m1, err := rc.reg.ManifestV1(repo, tag)
if err != nil {
logrus.WithFields(logrus.Fields{
"func": "vulnerabilities",
"URL": r.URL,
"method": r.Method,
"repo": repo,
"tag": tag,
}).Errorf("getting v1 manifest for %s:%s failed: %v", repo, tag, err)
w.WriteHeader(http.StatusNotFound)
fmt.Fprint(w, "Manifest not found")
return
}
result := clair.VulnerabilityReport{}
if rc.cl != nil {
result, err = rc.cl.Vulnerabilities(rc.reg, repo, tag, m1)
result, err = rc.cl.Vulnerabilities(rc.reg, repo, tag)
if err != nil {
logrus.WithFields(logrus.Fields{
"func": "vulnerabilities",

View file

@ -11,7 +11,7 @@ import (
"github.com/gorilla/mux"
"github.com/jessfraz/reg/clair"
"github.com/jessfraz/reg/registry"
"github.com/jessfraz/reg/utils"
"github.com/jessfraz/reg/repoutils"
wordwrap "github.com/mitchellh/go-wordwrap"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -98,7 +98,7 @@ func main() {
},
}
app.Action = func(c *cli.Context) error {
auth, err := utils.GetAuthConfig(c.GlobalString("username"), c.GlobalString("password"), c.GlobalString("registry"))
auth, err := repoutils.GetAuthConfig(c.GlobalString("username"), c.GlobalString("password"), c.GlobalString("registry"))
if err != nil {
logrus.Fatal(err)
}

28
tags.go Normal file
View file

@ -0,0 +1,28 @@
package main
import (
"fmt"
"strings"
"github.com/urfave/cli"
)
var tagsCommand = cli.Command{
Name: "tags",
Usage: "get the tags for a repository",
Action: func(c *cli.Context) error {
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
tags, err := r.Tags(c.Args()[0])
if err != nil {
return err
}
// Print the tags.
fmt.Println(strings.Join(tags, "\n"))
return nil
},
}

103
vulns.go Normal file
View file

@ -0,0 +1,103 @@
package main
import (
"errors"
"fmt"
"github.com/Sirupsen/logrus"
"github.com/jessfraz/reg/clair"
"github.com/jessfraz/reg/repoutils"
"github.com/urfave/cli"
)
var vulnsCommand = cli.Command{
Name: "vulns",
Usage: "get a vulnerability report for the image from CoreOS Clair",
Flags: []cli.Flag{
cli.StringFlag{
Name: "clair",
Usage: "url to clair instance",
},
cli.IntFlag{
Name: "fixable-threshold",
Usage: "number of fixable issues permitted",
Value: 0,
},
},
Action: func(c *cli.Context) error {
if c.String("clair") == "" {
return errors.New("clair url cannot be empty, pass --clair")
}
if c.Int("fixable-threshold") < 0 {
return errors.New("fixable threshold must be a positive integer")
}
if len(c.Args()) < 1 {
return fmt.Errorf("pass the name of the repository")
}
repo, ref, err := repoutils.GetRepoAndRef(c.Args()[0])
if err != nil {
return err
}
// Initialize clair client.
cr, err := clair.New(c.String("clair"), c.GlobalBool("debug"))
if err != nil {
return err
}
// Get the vulnerability report.
report, err := cr.Vulnerabilities(r, repo, ref)
if err != nil {
return err
}
// Iterate over the vulnerabilities by severity list.
for sev, vulns := range report.VulnsBySeverity {
for _, v := range vulns {
if sev == "Fixable" {
fmt.Printf("%s: [%s] \n%s\n%s\n", v.Name, v.Severity+" - Fixable", v.Description, v.Link)
fmt.Printf("Fixed by: %s\n", v.FixedBy)
} else {
fmt.Printf("%s: [%s] \n%s\n%s\n", v.Name, v.Severity, v.Description, v.Link)
}
fmt.Println("-----------------------------------------")
}
}
// Print summary and count.
for sev, vulns := range report.VulnsBySeverity {
fmt.Printf("%s: %d\n", sev, len(vulns))
}
// Return an error if there are more than 1 fixable vulns.
fixable, ok := report.VulnsBySeverity["Fixable"]
if ok {
if len(fixable) > c.Int("fixable-threshold") {
logrus.Fatalf("%d fixable vulnerabilities found", len(fixable))
}
}
// Return an error if there are more than 10 bad vulns.
badVulns := 0
// Include any high vulns.
if highVulns, ok := report.VulnsBySeverity["High"]; ok {
badVulns += len(highVulns)
}
// Include any critical vulns.
if criticalVulns, ok := report.VulnsBySeverity["Critical"]; ok {
badVulns += len(criticalVulns)
}
// Include any defcon1 vulns.
if defcon1Vulns, ok := report.VulnsBySeverity["Defcon1"]; ok {
badVulns += len(defcon1Vulns)
}
if badVulns > 10 {
logrus.Fatalf("%d bad vulnerabilities found", badVulns)
}
return nil
},
}