package container import ( "errors" "fmt" "net" "strconv" "strings" "time" "github.com/sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" enginecontainer "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/filters" enginemount "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" volumetypes "github.com/docker/docker/api/types/volume" "github.com/docker/docker/daemon/cluster/convert" executorpkg "github.com/docker/docker/daemon/cluster/executor" clustertypes "github.com/docker/docker/daemon/cluster/provider" "github.com/docker/go-connections/nat" netconst "github.com/docker/libnetwork/datastore" "github.com/docker/swarmkit/agent/exec" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/genericresource" "github.com/docker/swarmkit/template" gogotypes "github.com/gogo/protobuf/types" ) const ( // Explicitly use the kernel's default setting for CPU quota of 100ms. // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt cpuQuotaPeriod = 100 * time.Millisecond // systemLabelPrefix represents the reserved namespace for system labels. systemLabelPrefix = "com.docker.swarm" ) // containerConfig converts task properties into docker container compatible // components. type containerConfig struct { task *api.Task networksAttachments map[string]*api.NetworkAttachment } // newContainerConfig returns a validated container config. No methods should // return an error if this function returns without error. func newContainerConfig(t *api.Task, node *api.NodeDescription) (*containerConfig, error) { var c containerConfig return &c, c.setTask(t, node) } func (c *containerConfig) setTask(t *api.Task, node *api.NodeDescription) error { if t.Spec.GetContainer() == nil && t.Spec.GetAttachment() == nil { return exec.ErrRuntimeUnsupported } container := t.Spec.GetContainer() if container != nil { if container.Image == "" { return ErrImageRequired } if err := validateMounts(container.Mounts); err != nil { return err } } // index the networks by name c.networksAttachments = make(map[string]*api.NetworkAttachment, len(t.Networks)) for _, attachment := range t.Networks { c.networksAttachments[attachment.Network.Spec.Annotations.Name] = attachment } c.task = t if t.Spec.GetContainer() != nil { preparedSpec, err := template.ExpandContainerSpec(node, t) if err != nil { return err } c.task.Spec.Runtime = &api.TaskSpec_Container{ Container: preparedSpec, } } return nil } func (c *containerConfig) networkAttachmentContainerID() string { attachment := c.task.Spec.GetAttachment() if attachment == nil { return "" } return attachment.ContainerID } func (c *containerConfig) taskID() string { return c.task.ID } func (c *containerConfig) endpoint() *api.Endpoint { return c.task.Endpoint } func (c *containerConfig) spec() *api.ContainerSpec { return c.task.Spec.GetContainer() } func (c *containerConfig) nameOrID() string { if c.task.Spec.GetContainer() != nil { return c.name() } return c.networkAttachmentContainerID() } func (c *containerConfig) name() string { if c.task.Annotations.Name != "" { // if set, use the container Annotations.Name field, set in the orchestrator. return c.task.Annotations.Name } slot := fmt.Sprint(c.task.Slot) if slot == "" || c.task.Slot == 0 { slot = c.task.NodeID } // fallback to service.slot.id. return fmt.Sprintf("%s.%s.%s", c.task.ServiceAnnotations.Name, slot, c.task.ID) } func (c *containerConfig) image() string { raw := c.spec().Image ref, err := reference.ParseNormalizedNamed(raw) if err != nil { return raw } return reference.FamiliarString(reference.TagNameOnly(ref)) } func (c *containerConfig) portBindings() nat.PortMap { portBindings := nat.PortMap{} if c.task.Endpoint == nil { return portBindings } for _, portConfig := range c.task.Endpoint.Ports { if portConfig.PublishMode != api.PublishModeHost { continue } port := nat.Port(fmt.Sprintf("%d/%s", portConfig.TargetPort, strings.ToLower(portConfig.Protocol.String()))) binding := []nat.PortBinding{ {}, } if portConfig.PublishedPort != 0 { binding[0].HostPort = strconv.Itoa(int(portConfig.PublishedPort)) } portBindings[port] = binding } return portBindings } func (c *containerConfig) isolation() enginecontainer.Isolation { return convert.IsolationFromGRPC(c.spec().Isolation) } func (c *containerConfig) exposedPorts() map[nat.Port]struct{} { exposedPorts := make(map[nat.Port]struct{}) if c.task.Endpoint == nil { return exposedPorts } for _, portConfig := range c.task.Endpoint.Ports { if portConfig.PublishMode != api.PublishModeHost { continue } port := nat.Port(fmt.Sprintf("%d/%s", portConfig.TargetPort, strings.ToLower(portConfig.Protocol.String()))) exposedPorts[port] = struct{}{} } return exposedPorts } func (c *containerConfig) config() *enginecontainer.Config { genericEnvs := genericresource.EnvFormat(c.task.AssignedGenericResources, "DOCKER_RESOURCE") env := append(c.spec().Env, genericEnvs...) config := &enginecontainer.Config{ Labels: c.labels(), StopSignal: c.spec().StopSignal, Tty: c.spec().TTY, OpenStdin: c.spec().OpenStdin, User: c.spec().User, Env: env, Hostname: c.spec().Hostname, WorkingDir: c.spec().Dir, Image: c.image(), ExposedPorts: c.exposedPorts(), Healthcheck: c.healthcheck(), } if len(c.spec().Command) > 0 { // If Command is provided, we replace the whole invocation with Command // by replacing Entrypoint and specifying Cmd. Args is ignored in this // case. config.Entrypoint = append(config.Entrypoint, c.spec().Command...) config.Cmd = append(config.Cmd, c.spec().Args...) } else if len(c.spec().Args) > 0 { // In this case, we assume the image has an Entrypoint and Args // specifies the arguments for that entrypoint. config.Cmd = c.spec().Args } return config } func (c *containerConfig) labels() map[string]string { var ( system = map[string]string{ "task": "", // mark as cluster task "task.id": c.task.ID, "task.name": c.name(), "node.id": c.task.NodeID, "service.id": c.task.ServiceID, "service.name": c.task.ServiceAnnotations.Name, } labels = make(map[string]string) ) // base labels are those defined in the spec. for k, v := range c.spec().Labels { labels[k] = v } // we then apply the overrides from the task, which may be set via the // orchestrator. for k, v := range c.task.Annotations.Labels { labels[k] = v } // finally, we apply the system labels, which override all labels. for k, v := range system { labels[strings.Join([]string{systemLabelPrefix, k}, ".")] = v } return labels } func (c *containerConfig) mounts() []enginemount.Mount { var r []enginemount.Mount for _, mount := range c.spec().Mounts { r = append(r, convertMount(mount)) } return r } func convertMount(m api.Mount) enginemount.Mount { mount := enginemount.Mount{ Source: m.Source, Target: m.Target, ReadOnly: m.ReadOnly, } switch m.Type { case api.MountTypeBind: mount.Type = enginemount.TypeBind case api.MountTypeVolume: mount.Type = enginemount.TypeVolume case api.MountTypeTmpfs: mount.Type = enginemount.TypeTmpfs } if m.BindOptions != nil { mount.BindOptions = &enginemount.BindOptions{} switch m.BindOptions.Propagation { case api.MountPropagationRPrivate: mount.BindOptions.Propagation = enginemount.PropagationRPrivate case api.MountPropagationPrivate: mount.BindOptions.Propagation = enginemount.PropagationPrivate case api.MountPropagationRSlave: mount.BindOptions.Propagation = enginemount.PropagationRSlave case api.MountPropagationSlave: mount.BindOptions.Propagation = enginemount.PropagationSlave case api.MountPropagationRShared: mount.BindOptions.Propagation = enginemount.PropagationRShared case api.MountPropagationShared: mount.BindOptions.Propagation = enginemount.PropagationShared } } if m.VolumeOptions != nil { mount.VolumeOptions = &enginemount.VolumeOptions{ NoCopy: m.VolumeOptions.NoCopy, } if m.VolumeOptions.Labels != nil { mount.VolumeOptions.Labels = make(map[string]string, len(m.VolumeOptions.Labels)) for k, v := range m.VolumeOptions.Labels { mount.VolumeOptions.Labels[k] = v } } if m.VolumeOptions.DriverConfig != nil { mount.VolumeOptions.DriverConfig = &enginemount.Driver{ Name: m.VolumeOptions.DriverConfig.Name, } if m.VolumeOptions.DriverConfig.Options != nil { mount.VolumeOptions.DriverConfig.Options = make(map[string]string, len(m.VolumeOptions.DriverConfig.Options)) for k, v := range m.VolumeOptions.DriverConfig.Options { mount.VolumeOptions.DriverConfig.Options[k] = v } } } } if m.TmpfsOptions != nil { mount.TmpfsOptions = &enginemount.TmpfsOptions{ SizeBytes: m.TmpfsOptions.SizeBytes, Mode: m.TmpfsOptions.Mode, } } return mount } func (c *containerConfig) healthcheck() *enginecontainer.HealthConfig { hcSpec := c.spec().Healthcheck if hcSpec == nil { return nil } interval, _ := gogotypes.DurationFromProto(hcSpec.Interval) timeout, _ := gogotypes.DurationFromProto(hcSpec.Timeout) startPeriod, _ := gogotypes.DurationFromProto(hcSpec.StartPeriod) return &enginecontainer.HealthConfig{ Test: hcSpec.Test, Interval: interval, Timeout: timeout, Retries: int(hcSpec.Retries), StartPeriod: startPeriod, } } func (c *containerConfig) hostConfig() *enginecontainer.HostConfig { hc := &enginecontainer.HostConfig{ Resources: c.resources(), GroupAdd: c.spec().Groups, PortBindings: c.portBindings(), Mounts: c.mounts(), ReadonlyRootfs: c.spec().ReadOnly, Isolation: c.isolation(), } if c.spec().DNSConfig != nil { hc.DNS = c.spec().DNSConfig.Nameservers hc.DNSSearch = c.spec().DNSConfig.Search hc.DNSOptions = c.spec().DNSConfig.Options } c.applyPrivileges(hc) // The format of extra hosts on swarmkit is specified in: // http://man7.org/linux/man-pages/man5/hosts.5.html // IP_address canonical_hostname [aliases...] // However, the format of ExtraHosts in HostConfig is // : // We need to do the conversion here // (Alias is ignored for now) for _, entry := range c.spec().Hosts { parts := strings.Fields(entry) if len(parts) > 1 { hc.ExtraHosts = append(hc.ExtraHosts, fmt.Sprintf("%s:%s", parts[1], parts[0])) } } if c.task.LogDriver != nil { hc.LogConfig = enginecontainer.LogConfig{ Type: c.task.LogDriver.Name, Config: c.task.LogDriver.Options, } } if len(c.task.Networks) > 0 { labels := c.task.Networks[0].Network.Spec.Annotations.Labels name := c.task.Networks[0].Network.Spec.Annotations.Name if v, ok := labels["com.docker.swarm.predefined"]; ok && v == "true" { hc.NetworkMode = enginecontainer.NetworkMode(name) } } return hc } // This handles the case of volumes that are defined inside a service Mount func (c *containerConfig) volumeCreateRequest(mount *api.Mount) *volumetypes.VolumesCreateBody { var ( driverName string driverOpts map[string]string labels map[string]string ) if mount.VolumeOptions != nil && mount.VolumeOptions.DriverConfig != nil { driverName = mount.VolumeOptions.DriverConfig.Name driverOpts = mount.VolumeOptions.DriverConfig.Options labels = mount.VolumeOptions.Labels } if mount.VolumeOptions != nil { return &volumetypes.VolumesCreateBody{ Name: mount.Source, Driver: driverName, DriverOpts: driverOpts, Labels: labels, } } return nil } func (c *containerConfig) resources() enginecontainer.Resources { resources := enginecontainer.Resources{} // If no limits are specified let the engine use its defaults. // // TODO(aluzzardi): We might want to set some limits anyway otherwise // "unlimited" tasks will step over the reservation of other tasks. r := c.task.Spec.Resources if r == nil || r.Limits == nil { return resources } if r.Limits.MemoryBytes > 0 { resources.Memory = r.Limits.MemoryBytes } if r.Limits.NanoCPUs > 0 { // CPU Period must be set in microseconds. resources.CPUPeriod = int64(cpuQuotaPeriod / time.Microsecond) resources.CPUQuota = r.Limits.NanoCPUs * resources.CPUPeriod / 1e9 } return resources } // Docker daemon supports just 1 network during container create. func (c *containerConfig) createNetworkingConfig(b executorpkg.Backend) *network.NetworkingConfig { var networks []*api.NetworkAttachment if c.task.Spec.GetContainer() != nil || c.task.Spec.GetAttachment() != nil { networks = c.task.Networks } epConfig := make(map[string]*network.EndpointSettings) if len(networks) > 0 { epConfig[networks[0].Network.Spec.Annotations.Name] = getEndpointConfig(networks[0], b) } return &network.NetworkingConfig{EndpointsConfig: epConfig} } // TODO: Merge this function with createNetworkingConfig after daemon supports multiple networks in container create func (c *containerConfig) connectNetworkingConfig(b executorpkg.Backend) *network.NetworkingConfig { var networks []*api.NetworkAttachment if c.task.Spec.GetContainer() != nil { networks = c.task.Networks } // First network is used during container create. Other networks are used in "docker network connect" if len(networks) < 2 { return nil } epConfig := make(map[string]*network.EndpointSettings) for _, na := range networks[1:] { epConfig[na.Network.Spec.Annotations.Name] = getEndpointConfig(na, b) } return &network.NetworkingConfig{EndpointsConfig: epConfig} } func getEndpointConfig(na *api.NetworkAttachment, b executorpkg.Backend) *network.EndpointSettings { var ipv4, ipv6 string for _, addr := range na.Addresses { ip, _, err := net.ParseCIDR(addr) if err != nil { continue } if ip.To4() != nil { ipv4 = ip.String() continue } if ip.To16() != nil { ipv6 = ip.String() } } n := &network.EndpointSettings{ NetworkID: na.Network.ID, IPAMConfig: &network.EndpointIPAMConfig{ IPv4Address: ipv4, IPv6Address: ipv6, }, DriverOpts: na.DriverAttachmentOpts, } if v, ok := na.Network.Spec.Annotations.Labels["com.docker.swarm.predefined"]; ok && v == "true" { if ln, err := b.FindNetwork(na.Network.Spec.Annotations.Name); err == nil { n.NetworkID = ln.ID() } } return n } func (c *containerConfig) virtualIP(networkID string) string { if c.task.Endpoint == nil { return "" } for _, eVip := range c.task.Endpoint.VirtualIPs { // We only support IPv4 VIPs for now. if eVip.NetworkID == networkID { vip, _, err := net.ParseCIDR(eVip.Addr) if err != nil { return "" } return vip.String() } } return "" } func (c *containerConfig) serviceConfig() *clustertypes.ServiceConfig { if len(c.task.Networks) == 0 { return nil } logrus.Debugf("Creating service config in agent for t = %+v", c.task) svcCfg := &clustertypes.ServiceConfig{ Name: c.task.ServiceAnnotations.Name, Aliases: make(map[string][]string), ID: c.task.ServiceID, VirtualAddresses: make(map[string]*clustertypes.VirtualAddress), } for _, na := range c.task.Networks { svcCfg.VirtualAddresses[na.Network.ID] = &clustertypes.VirtualAddress{ // We support only IPv4 virtual IP for now. IPv4: c.virtualIP(na.Network.ID), } if len(na.Aliases) > 0 { svcCfg.Aliases[na.Network.ID] = na.Aliases } } if c.task.Endpoint != nil { for _, ePort := range c.task.Endpoint.Ports { if ePort.PublishMode != api.PublishModeIngress { continue } svcCfg.ExposedPorts = append(svcCfg.ExposedPorts, &clustertypes.PortConfig{ Name: ePort.Name, Protocol: int32(ePort.Protocol), TargetPort: ePort.TargetPort, PublishedPort: ePort.PublishedPort, }) } } return svcCfg } // networks returns a list of network names attached to the container. The // returned name can be used to lookup the corresponding network create // options. func (c *containerConfig) networks() []string { var networks []string for name := range c.networksAttachments { networks = append(networks, name) } return networks } func (c *containerConfig) networkCreateRequest(name string) (clustertypes.NetworkCreateRequest, error) { na, ok := c.networksAttachments[name] if !ok { return clustertypes.NetworkCreateRequest{}, errors.New("container: unknown network referenced") } options := types.NetworkCreate{ // ID: na.Network.ID, Labels: na.Network.Spec.Annotations.Labels, Internal: na.Network.Spec.Internal, Attachable: na.Network.Spec.Attachable, Ingress: convert.IsIngressNetwork(na.Network), EnableIPv6: na.Network.Spec.Ipv6Enabled, CheckDuplicate: true, Scope: netconst.SwarmScope, } if na.Network.Spec.GetNetwork() != "" { options.ConfigFrom = &network.ConfigReference{ Network: na.Network.Spec.GetNetwork(), } } if na.Network.DriverState != nil { options.Driver = na.Network.DriverState.Name options.Options = na.Network.DriverState.Options } if na.Network.IPAM != nil { options.IPAM = &network.IPAM{ Driver: na.Network.IPAM.Driver.Name, Options: na.Network.IPAM.Driver.Options, } for _, ic := range na.Network.IPAM.Configs { c := network.IPAMConfig{ Subnet: ic.Subnet, IPRange: ic.Range, Gateway: ic.Gateway, } options.IPAM.Config = append(options.IPAM.Config, c) } } return clustertypes.NetworkCreateRequest{ ID: na.Network.ID, NetworkCreateRequest: types.NetworkCreateRequest{ Name: name, NetworkCreate: options, }, }, nil } func (c *containerConfig) applyPrivileges(hc *enginecontainer.HostConfig) { privileges := c.spec().Privileges if privileges == nil { return } credentials := privileges.CredentialSpec if credentials != nil { switch credentials.Source.(type) { case *api.Privileges_CredentialSpec_File: hc.SecurityOpt = append(hc.SecurityOpt, "credentialspec=file://"+credentials.GetFile()) case *api.Privileges_CredentialSpec_Registry: hc.SecurityOpt = append(hc.SecurityOpt, "credentialspec=registry://"+credentials.GetRegistry()) } } selinux := privileges.SELinuxContext if selinux != nil { if selinux.Disable { hc.SecurityOpt = append(hc.SecurityOpt, "label=disable") } if selinux.User != "" { hc.SecurityOpt = append(hc.SecurityOpt, "label=user:"+selinux.User) } if selinux.Role != "" { hc.SecurityOpt = append(hc.SecurityOpt, "label=role:"+selinux.Role) } if selinux.Level != "" { hc.SecurityOpt = append(hc.SecurityOpt, "label=level:"+selinux.Level) } if selinux.Type != "" { hc.SecurityOpt = append(hc.SecurityOpt, "label=type:"+selinux.Type) } } } func (c containerConfig) eventFilter() filters.Args { filter := filters.NewArgs() filter.Add("type", events.ContainerEventType) filter.Add("name", c.name()) filter.Add("label", fmt.Sprintf("%v.task.id=%v", systemLabelPrefix, c.task.ID)) return filter }