mirror of
https://github.com/genuinetools/reg.git
synced 2024-08-21 14:20:23 -04:00
559 lines
15 KiB
Go
559 lines
15 KiB
Go
package dockerfile // import "github.com/docker/docker/builder/dockerfile"
|
|
|
|
import (
|
|
"archive/tar"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/docker/docker/builder"
|
|
"github.com/docker/docker/builder/remotecontext"
|
|
"github.com/docker/docker/pkg/archive"
|
|
"github.com/docker/docker/pkg/containerfs"
|
|
"github.com/docker/docker/pkg/idtools"
|
|
"github.com/docker/docker/pkg/ioutils"
|
|
"github.com/docker/docker/pkg/progress"
|
|
"github.com/docker/docker/pkg/streamformatter"
|
|
"github.com/docker/docker/pkg/system"
|
|
"github.com/docker/docker/pkg/urlutil"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
const unnamedFilename = "__unnamed__"
|
|
|
|
type pathCache interface {
|
|
Load(key interface{}) (value interface{}, ok bool)
|
|
Store(key, value interface{})
|
|
}
|
|
|
|
// copyInfo is a data object which stores the metadata about each source file in
|
|
// a copyInstruction
|
|
type copyInfo struct {
|
|
root containerfs.ContainerFS
|
|
path string
|
|
hash string
|
|
noDecompress bool
|
|
}
|
|
|
|
func (c copyInfo) fullPath() (string, error) {
|
|
return c.root.ResolveScopedPath(c.path, true)
|
|
}
|
|
|
|
func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo {
|
|
return copyInfo{root: source.Root(), path: path, hash: hash}
|
|
}
|
|
|
|
func newCopyInfos(copyInfos ...copyInfo) []copyInfo {
|
|
return copyInfos
|
|
}
|
|
|
|
// copyInstruction is a fully parsed COPY or ADD command that is passed to
|
|
// Builder.performCopy to copy files into the image filesystem
|
|
type copyInstruction struct {
|
|
cmdName string
|
|
infos []copyInfo
|
|
dest string
|
|
chownStr string
|
|
allowLocalDecompression bool
|
|
}
|
|
|
|
// copier reads a raw COPY or ADD command, fetches remote sources using a downloader,
|
|
// and creates a copyInstruction
|
|
type copier struct {
|
|
imageSource *imageMount
|
|
source builder.Source
|
|
pathCache pathCache
|
|
download sourceDownloader
|
|
platform string
|
|
// for cleanup. TODO: having copier.cleanup() is error prone and hard to
|
|
// follow. Code calling performCopy should manage the lifecycle of its params.
|
|
// Copier should take override source as input, not imageMount.
|
|
activeLayer builder.RWLayer
|
|
tmpPaths []string
|
|
}
|
|
|
|
func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier {
|
|
return copier{
|
|
source: req.source,
|
|
pathCache: req.builder.pathCache,
|
|
download: download,
|
|
imageSource: imageSource,
|
|
platform: req.builder.options.Platform,
|
|
}
|
|
}
|
|
|
|
func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstruction, error) {
|
|
inst := copyInstruction{cmdName: cmdName}
|
|
last := len(args) - 1
|
|
|
|
// Work in platform-specific filepath semantics
|
|
inst.dest = fromSlash(args[last], o.platform)
|
|
separator := string(separator(o.platform))
|
|
infos, err := o.getCopyInfosForSourcePaths(args[0:last], inst.dest)
|
|
if err != nil {
|
|
return inst, errors.Wrapf(err, "%s failed", cmdName)
|
|
}
|
|
if len(infos) > 1 && !strings.HasSuffix(inst.dest, separator) {
|
|
return inst, errors.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
|
|
}
|
|
inst.infos = infos
|
|
return inst, nil
|
|
}
|
|
|
|
// getCopyInfosForSourcePaths iterates over the source files and calculate the info
|
|
// needed to copy (e.g. hash value if cached)
|
|
// The dest is used in case source is URL (and ends with "/")
|
|
func (o *copier) getCopyInfosForSourcePaths(sources []string, dest string) ([]copyInfo, error) {
|
|
var infos []copyInfo
|
|
for _, orig := range sources {
|
|
subinfos, err := o.getCopyInfoForSourcePath(orig, dest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
infos = append(infos, subinfos...)
|
|
}
|
|
|
|
if len(infos) == 0 {
|
|
return nil, errors.New("no source files were specified")
|
|
}
|
|
return infos, nil
|
|
}
|
|
|
|
func (o *copier) getCopyInfoForSourcePath(orig, dest string) ([]copyInfo, error) {
|
|
if !urlutil.IsURL(orig) {
|
|
return o.calcCopyInfo(orig, true)
|
|
}
|
|
|
|
remote, path, err := o.download(orig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// If path == "" then we are unable to determine filename from src
|
|
// We have to make sure dest is available
|
|
if path == "" {
|
|
if strings.HasSuffix(dest, "/") {
|
|
return nil, errors.Errorf("cannot determine filename for source %s", orig)
|
|
}
|
|
path = unnamedFilename
|
|
}
|
|
o.tmpPaths = append(o.tmpPaths, remote.Root().Path())
|
|
|
|
hash, err := remote.Hash(path)
|
|
ci := newCopyInfoFromSource(remote, path, hash)
|
|
ci.noDecompress = true // data from http shouldn't be extracted even on ADD
|
|
return newCopyInfos(ci), err
|
|
}
|
|
|
|
// Cleanup removes any temporary directories created as part of downloading
|
|
// remote files.
|
|
func (o *copier) Cleanup() {
|
|
for _, path := range o.tmpPaths {
|
|
os.RemoveAll(path)
|
|
}
|
|
o.tmpPaths = []string{}
|
|
if o.activeLayer != nil {
|
|
o.activeLayer.Release()
|
|
o.activeLayer = nil
|
|
}
|
|
}
|
|
|
|
// TODO: allowWildcards can probably be removed by refactoring this function further.
|
|
func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, error) {
|
|
imageSource := o.imageSource
|
|
|
|
// TODO: do this when creating copier. Requires validateCopySourcePath
|
|
// (and other below) to be aware of the difference sources. Why is it only
|
|
// done on image Source?
|
|
if imageSource != nil {
|
|
var err error
|
|
rwLayer, err := imageSource.NewRWLayer()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
o.activeLayer = rwLayer
|
|
|
|
o.source, err = remotecontext.NewLazySource(rwLayer.Root())
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "failed to create context for copy from %s", rwLayer.Root().Path())
|
|
}
|
|
}
|
|
|
|
if o.source == nil {
|
|
return nil, errors.Errorf("missing build context")
|
|
}
|
|
|
|
root := o.source.Root()
|
|
|
|
if err := validateCopySourcePath(imageSource, origPath, root.OS()); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Work in source OS specific filepath semantics
|
|
// For LCOW, this is NOT the daemon OS.
|
|
origPath = root.FromSlash(origPath)
|
|
origPath = strings.TrimPrefix(origPath, string(root.Separator()))
|
|
origPath = strings.TrimPrefix(origPath, "."+string(root.Separator()))
|
|
|
|
// Deal with wildcards
|
|
if allowWildcards && containsWildcards(origPath, root.OS()) {
|
|
return o.copyWithWildcards(origPath)
|
|
}
|
|
|
|
if imageSource != nil && imageSource.ImageID() != "" {
|
|
// return a cached copy if one exists
|
|
if h, ok := o.pathCache.Load(imageSource.ImageID() + origPath); ok {
|
|
return newCopyInfos(newCopyInfoFromSource(o.source, origPath, h.(string))), nil
|
|
}
|
|
}
|
|
|
|
// Deal with the single file case
|
|
copyInfo, err := copyInfoForFile(o.source, origPath)
|
|
switch {
|
|
case err != nil:
|
|
return nil, err
|
|
case copyInfo.hash != "":
|
|
o.storeInPathCache(imageSource, origPath, copyInfo.hash)
|
|
return newCopyInfos(copyInfo), err
|
|
}
|
|
|
|
// TODO: remove, handle dirs in Hash()
|
|
subfiles, err := walkSource(o.source, origPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
hash := hashStringSlice("dir", subfiles)
|
|
o.storeInPathCache(imageSource, origPath, hash)
|
|
return newCopyInfos(newCopyInfoFromSource(o.source, origPath, hash)), nil
|
|
}
|
|
|
|
func containsWildcards(name, platform string) bool {
|
|
isWindows := platform == "windows"
|
|
for i := 0; i < len(name); i++ {
|
|
ch := name[i]
|
|
if ch == '\\' && !isWindows {
|
|
i++
|
|
} else if ch == '*' || ch == '?' || ch == '[' {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (o *copier) storeInPathCache(im *imageMount, path string, hash string) {
|
|
if im != nil {
|
|
o.pathCache.Store(im.ImageID()+path, hash)
|
|
}
|
|
}
|
|
|
|
func (o *copier) copyWithWildcards(origPath string) ([]copyInfo, error) {
|
|
root := o.source.Root()
|
|
var copyInfos []copyInfo
|
|
if err := root.Walk(root.Path(), func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel, err := remotecontext.Rel(root, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if rel == "." {
|
|
return nil
|
|
}
|
|
if match, _ := root.Match(origPath, rel); !match {
|
|
return nil
|
|
}
|
|
|
|
// Note we set allowWildcards to false in case the name has
|
|
// a * in it
|
|
subInfos, err := o.calcCopyInfo(rel, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
copyInfos = append(copyInfos, subInfos...)
|
|
return nil
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
return copyInfos, nil
|
|
}
|
|
|
|
func copyInfoForFile(source builder.Source, path string) (copyInfo, error) {
|
|
fi, err := remotecontext.StatAt(source, path)
|
|
if err != nil {
|
|
return copyInfo{}, err
|
|
}
|
|
|
|
if fi.IsDir() {
|
|
return copyInfo{}, nil
|
|
}
|
|
hash, err := source.Hash(path)
|
|
if err != nil {
|
|
return copyInfo{}, err
|
|
}
|
|
return newCopyInfoFromSource(source, path, "file:"+hash), nil
|
|
}
|
|
|
|
// TODO: dedupe with copyWithWildcards()
|
|
func walkSource(source builder.Source, origPath string) ([]string, error) {
|
|
fp, err := remotecontext.FullPath(source, origPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Must be a dir
|
|
var subfiles []string
|
|
err = source.Root().Walk(fp, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel, err := remotecontext.Rel(source.Root(), path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if rel == "." {
|
|
return nil
|
|
}
|
|
hash, err := source.Hash(rel)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
// we already checked handleHash above
|
|
subfiles = append(subfiles, hash)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sort.Strings(subfiles)
|
|
return subfiles, nil
|
|
}
|
|
|
|
type sourceDownloader func(string) (builder.Source, string, error)
|
|
|
|
func newRemoteSourceDownloader(output, stdout io.Writer) sourceDownloader {
|
|
return func(url string) (builder.Source, string, error) {
|
|
return downloadSource(output, stdout, url)
|
|
}
|
|
}
|
|
|
|
func errOnSourceDownload(_ string) (builder.Source, string, error) {
|
|
return nil, "", errors.New("source can't be a URL for COPY")
|
|
}
|
|
|
|
func getFilenameForDownload(path string, resp *http.Response) string {
|
|
// Guess filename based on source
|
|
if path != "" && !strings.HasSuffix(path, "/") {
|
|
if filename := filepath.Base(filepath.FromSlash(path)); filename != "" {
|
|
return filename
|
|
}
|
|
}
|
|
|
|
// Guess filename based on Content-Disposition
|
|
if contentDisposition := resp.Header.Get("Content-Disposition"); contentDisposition != "" {
|
|
if _, params, err := mime.ParseMediaType(contentDisposition); err == nil {
|
|
if params["filename"] != "" && !strings.HasSuffix(params["filename"], "/") {
|
|
if filename := filepath.Base(filepath.FromSlash(params["filename"])); filename != "" {
|
|
return filename
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote builder.Source, p string, err error) {
|
|
u, err := url.Parse(srcURL)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
resp, err := remotecontext.GetWithStatusError(srcURL)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
filename := getFilenameForDownload(u.Path, resp)
|
|
|
|
// Prepare file in a tmp dir
|
|
tmpDir, err := ioutils.TempDir("", "docker-remote")
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer func() {
|
|
if err != nil {
|
|
os.RemoveAll(tmpDir)
|
|
}
|
|
}()
|
|
// If filename is empty, the returned filename will be "" but
|
|
// the tmp filename will be created as "__unnamed__"
|
|
tmpFileName := filename
|
|
if filename == "" {
|
|
tmpFileName = unnamedFilename
|
|
}
|
|
tmpFileName = filepath.Join(tmpDir, tmpFileName)
|
|
tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
progressOutput := streamformatter.NewJSONProgressOutput(output, true)
|
|
progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading")
|
|
// Download and dump result to tmp file
|
|
// TODO: add filehash directly
|
|
if _, err = io.Copy(tmpFile, progressReader); err != nil {
|
|
tmpFile.Close()
|
|
return
|
|
}
|
|
// TODO: how important is this random blank line to the output?
|
|
fmt.Fprintln(stdout)
|
|
|
|
// Set the mtime to the Last-Modified header value if present
|
|
// Otherwise just remove atime and mtime
|
|
mTime := time.Time{}
|
|
|
|
lastMod := resp.Header.Get("Last-Modified")
|
|
if lastMod != "" {
|
|
// If we can't parse it then just let it default to 'zero'
|
|
// otherwise use the parsed time value
|
|
if parsedMTime, err := http.ParseTime(lastMod); err == nil {
|
|
mTime = parsedMTime
|
|
}
|
|
}
|
|
|
|
tmpFile.Close()
|
|
|
|
if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil {
|
|
return
|
|
}
|
|
|
|
lc, err := remotecontext.NewLazySource(containerfs.NewLocalContainerFS(tmpDir))
|
|
return lc, filename, err
|
|
}
|
|
|
|
type copyFileOptions struct {
|
|
decompress bool
|
|
chownPair idtools.IDPair
|
|
archiver Archiver
|
|
}
|
|
|
|
type copyEndpoint struct {
|
|
driver containerfs.Driver
|
|
path string
|
|
}
|
|
|
|
func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) error {
|
|
srcPath, err := source.fullPath()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
destPath, err := dest.fullPath()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
archiver := options.archiver
|
|
|
|
srcEndpoint := ©Endpoint{driver: source.root, path: srcPath}
|
|
destEndpoint := ©Endpoint{driver: dest.root, path: destPath}
|
|
|
|
src, err := source.root.Stat(srcPath)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "source path not found")
|
|
}
|
|
if src.IsDir() {
|
|
return copyDirectory(archiver, srcEndpoint, destEndpoint, options.chownPair)
|
|
}
|
|
if options.decompress && isArchivePath(source.root, srcPath) && !source.noDecompress {
|
|
return archiver.UntarPath(srcPath, destPath)
|
|
}
|
|
|
|
destExistsAsDir, err := isExistingDirectory(destEndpoint)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// dest.path must be used because destPath has already been cleaned of any
|
|
// trailing slash
|
|
if endsInSlash(dest.root, dest.path) || destExistsAsDir {
|
|
// source.path must be used to get the correct filename when the source
|
|
// is a symlink
|
|
destPath = dest.root.Join(destPath, source.root.Base(source.path))
|
|
destEndpoint = ©Endpoint{driver: dest.root, path: destPath}
|
|
}
|
|
return copyFile(archiver, srcEndpoint, destEndpoint, options.chownPair)
|
|
}
|
|
|
|
func isArchivePath(driver containerfs.ContainerFS, path string) bool {
|
|
file, err := driver.Open(path)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
defer file.Close()
|
|
rdr, err := archive.DecompressStream(file)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
r := tar.NewReader(rdr)
|
|
_, err = r.Next()
|
|
return err == nil
|
|
}
|
|
|
|
func copyDirectory(archiver Archiver, source, dest *copyEndpoint, chownPair idtools.IDPair) error {
|
|
destExists, err := isExistingDirectory(dest)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to query destination path")
|
|
}
|
|
|
|
if err := archiver.CopyWithTar(source.path, dest.path); err != nil {
|
|
return errors.Wrapf(err, "failed to copy directory")
|
|
}
|
|
// TODO: @gupta-ak. Investigate how LCOW permission mappings will work.
|
|
return fixPermissions(source.path, dest.path, chownPair, !destExists)
|
|
}
|
|
|
|
func copyFile(archiver Archiver, source, dest *copyEndpoint, chownPair idtools.IDPair) error {
|
|
if runtime.GOOS == "windows" && dest.driver.OS() == "linux" {
|
|
// LCOW
|
|
if err := dest.driver.MkdirAll(dest.driver.Dir(dest.path), 0755); err != nil {
|
|
return errors.Wrapf(err, "failed to create new directory")
|
|
}
|
|
} else {
|
|
if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest.path), 0755, chownPair); err != nil {
|
|
// Normal containers
|
|
return errors.Wrapf(err, "failed to create new directory")
|
|
}
|
|
}
|
|
|
|
if err := archiver.CopyFileWithTar(source.path, dest.path); err != nil {
|
|
return errors.Wrapf(err, "failed to copy file")
|
|
}
|
|
// TODO: @gupta-ak. Investigate how LCOW permission mappings will work.
|
|
return fixPermissions(source.path, dest.path, chownPair, false)
|
|
}
|
|
|
|
func endsInSlash(driver containerfs.Driver, path string) bool {
|
|
return strings.HasSuffix(path, string(driver.Separator()))
|
|
}
|
|
|
|
// isExistingDirectory returns true if the path exists and is a directory
|
|
func isExistingDirectory(point *copyEndpoint) (bool, error) {
|
|
destStat, err := point.driver.Stat(point.path)
|
|
switch {
|
|
case os.IsNotExist(err):
|
|
return false, nil
|
|
case err != nil:
|
|
return false, err
|
|
}
|
|
return destStat.IsDir(), nil
|
|
}
|