// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. // isula-build licensed under the Mulan PSL v2. // You can use this software according to the terms and conditions of the Mulan PSL v2. // You may obtain a copy of Mulan PSL v2 at: // http://license.coscl.org.cn/MulanPSL2 // THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR // PURPOSE. // See the Mulan PSL v2 for more details. // Author: Zhongkai Lei // Create: 2020-03-20 // Description: image related functions // Package image includes image related functions package image import ( "context" "encoding/json" "fmt" "io" "os" "path" "runtime" "strings" "time" "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/image" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/sysregistriesv2" "github.com/containers/image/v5/signature" is "github.com/containers/image/v5/storage" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" "github.com/containers/storage" "github.com/containers/storage/pkg/stringid" "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" constant "isula.org/isula-build" dockerfile "isula.org/isula-build/builder/dockerfile/parser" "isula.org/isula-build/pkg/docker" "isula.org/isula-build/store" "isula.org/isula-build/util" ) var ( // DefaultRegistryConfigPath describes the config path of registries DefaultRegistryConfigPath = "/etc/isula-build/registries.toml" // DefaultSignaturePolicyPath describes the policy path DefaultSignaturePolicyPath = "/etc/isula-build/policy.json" // DefaultRegistryDirPath is the dir to store registry configs DefaultRegistryDirPath = "/etc/isula-build/registries.d" // DefaultBlobInfoCacheDirPath is the dir to cache blob info DefaultBlobInfoCacheDirPath = "/var/lib/isula-build" // DefaultAuthFile is authentication file used for registry connection DefaultAuthFile = "/etc/isula-build/auth.json" ) // PrepareImageOptions describes the options required for preparing the image type PrepareImageOptions struct { SystemContext *types.SystemContext Ctx context.Context FromImage string Store store.Store Reporter io.Writer } // ContainerDescribe describes the contents for container type ContainerDescribe struct { ContainerName string ContainerID string Mountpoint string } // Describe describes the prepared image type Describe struct { ContainerDesc *ContainerDescribe Image types.Image ImageID string TopLayID string } type pullOption struct { sc *types.SystemContext ctx context.Context reporter io.Writer srcRef types.ImageReference dstRef types.ImageReference dstName string } func pullImage(opt pullOption) (types.ImageReference, error) { pLog := logrus.WithField(util.LogKeySessionID, opt.ctx.Value(util.LogFieldKey(util.LogKeySessionID))) policy, err := signature.DefaultPolicy(opt.sc) if err != nil { return nil, errors.Wrapf(err, "error obtaining default signature policy") } policyContext, err := signature.NewPolicyContext(policy) if err != nil { return nil, errors.Wrapf(err, "error creating new signature policy context") } defer func() { if err2 := policyContext.Destroy(); err2 != nil { pLog.Debugf("Error destroying signature policy context: %v", err2) } }() cpOpt := ©.Options{ ReportWriter: opt.reporter, SourceCtx: opt.sc, DestinationCtx: GetSystemContext(), } pLog.Debugf("Copying %q to %q", transports.ImageName(opt.srcRef), opt.dstName) if _, err := copy.Image(opt.ctx, policyContext, opt.dstRef, opt.srcRef, cpOpt); err != nil { pLog.Debugf("Error copying src image [%q] to dest image [%q] err: %v", transports.ImageName(opt.srcRef), opt.dstName, err) return nil, err } return opt.dstRef, nil } func pullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *storage.Image, error) { pLog := logrus.WithField(util.LogKeySessionID, opt.Ctx.Value(util.LogFieldKey(util.LogKeySessionID))) candidates, transport, err := ResolveName(opt.FromImage, opt.SystemContext, opt.Store) if err != nil { return nil, nil, errors.Wrapf(err, "error parsing reference to image %q", opt.FromImage) } if transport == "" { // if the image can be obtained from the local storage by image id, // then only one image can be obtained. if len(candidates) != 1 { return nil, nil, errors.New("transport is empty and multi or no image be found") } img, err := opt.Store.Image(candidates[0]) if err != nil { pLog.Errorf("Failed to find the image %q in local store: %v", candidates[0], err) return nil, nil, err } ref, err := is.Transport.ParseStoreReference(opt.Store, img.ID) if err != nil { return nil, nil, errors.Wrapf(err, "failed to get the ref in store by %q", candidates[0]) } pLog.Infof("Get image from local store first search by %q", opt.FromImage) return ref, img, nil } // record the last pull error var errPull error const tagSeperator = ":" for _, strImage := range candidates { if transport != util.DefaultTransport { transport += tagSeperator } srcRef, err := alltransports.ParseImageName(transport + strImage) if err != nil { pLog.Debugf("Failed to parse the image %q: %v", transport+strImage, err) continue } destImage, err := getLocalImageNameFromRef(opt.Store, srcRef) if err != nil { pLog.Debugf("Failed to get local image name for %q: %v", transport+strImage, err) continue } destRef, err := is.Transport.ParseStoreReference(opt.Store, destImage) if err != nil { pLog.Debugf("Failed to parse store reference for %q: %v", destImage, err) continue } img, err := is.Transport.GetStoreImage(opt.Store, destRef) if err == nil { // find the unique image in local store by name or digest pLog.Infof("Get image from local store second search by %q", opt.FromImage) return destRef, img, nil } // can not find image in local store, pull from registry pulledRef, err := pullImage(pullOption{ ctx: opt.Ctx, reporter: opt.Reporter, sc: opt.SystemContext, srcRef: srcRef, dstRef: destRef, dstName: destImage, }) if err != nil { pLog.Debugf("Failed to pull image %q: %v", transport+strImage, err) continue } pulledImg, err := is.Transport.GetStoreImage(opt.Store, pulledRef) if err != nil { errPull = err pLog.Infof("Failed to obtaining pulled image %q: %v", transports.ImageName(srcRef), err) continue } return pulledRef, pulledImg, nil } return nil, nil, errors.Errorf("failed to get the image in %#v: %v", candidates, errPull) } func instantiatingImage(ctx context.Context, sc *types.SystemContext, ref types.ImageReference) (types.Image, error) { imgSource, err := ref.NewImageSource(ctx, sc) if err != nil { return nil, errors.Wrapf(err, "instantiating image %q failed", transports.ImageName(ref)) } defer func() { if cerr := imgSource.Close(); cerr != nil { logrus.Warningf("Closing imgSource failed: %v", cerr) } }() byteManifest, mType, err := imgSource.GetManifest(ctx, nil) if err != nil { return nil, errors.Wrapf(err, "loading image %q manifest failed", transports.ImageName(ref)) } var ( instanceDigest *digest.Digest list manifest.List instance digest.Digest ) if manifest.MIMETypeIsMultiImage(mType) { list, err = manifest.ListFromBlob(byteManifest, mType) if err != nil { return nil, errors.Wrapf(err, "parsing image %q manifest as list failed", transports.ImageName(ref)) } instance, err = list.ChooseInstance(sc) if err != nil { return nil, errors.Wrapf(err, "finding the image in manifest list %q failed", transports.ImageName(ref)) } instanceDigest = &instance } baseImg, err := image.FromUnparsedImage(ctx, sc, image.UnparsedInstance(imgSource, instanceDigest)) if err != nil { return nil, errors.Wrapf(err, "instantiating image %q with instance %q failed", transports.ImageName(ref), instanceDigest) } return baseImg, nil } func getLocalImageNameFromRef(store storage.Store, srcRef types.ImageReference) (string, error) { if srcRef == nil { return "", errors.Errorf("reference to image is empty") } if srcRef.Transport().Name() == "docker-archive" { return stringid.GenerateRandomID() + ":" + stringid.GenerateRandomID(), nil } if srcRef.Transport().Name() != "docker" { return "", errors.Errorf("the %s transport is not supported yet", srcRef.Transport().Name()) } var name string ref := srcRef.DockerReference() if ref == nil { return "", errors.New("get the docker reference associated with source reference failed") } if named, ok := ref.(reference.Named); ok { name = named.Name() if tag, ok := ref.(reference.NamedTagged); ok { name = name + ":" + tag.Tag() } if dig, ok := ref.(reference.Canonical); ok { name = name + "@" + dig.Digest().String() } } if _, err := is.Transport.ParseStoreReference(store, name); err != nil { return "", errors.Wrapf(err, "parsing image name %q failed", name) } return name, nil } func createScratchV2Image() *docker.Image { return &docker.Image{ V1Image: docker.V1Image{ ContainerConfig: docker.Config{}, Config: &docker.Config{ ExposedPorts: make(docker.PortSet), Env: make([]string, 0, 0), Cmd: make([]string, 0, 0), Healthcheck: &docker.HealthConfig{}, Volumes: make(map[string]struct{}), Entrypoint: make([]string, 0, 0), OnBuild: make([]string, 0, 0), Labels: make(map[string]string), StopTimeout: nil, Shell: make([]string, 0, 0), }, }, RootFS: &docker.RootFS{}, History: make([]docker.History, 0, 0), } } func createImageV2Image(ctx context.Context, fromImage types.Image, targetMIMEType string) (*docker.Image, error) { imageName := transports.ImageName(fromImage.Reference()) _, imageMIMEType, err := fromImage.Manifest(ctx) if err != nil { return nil, errors.Wrapf(err, "failed to get MIME type from %q", imageName) } if targetMIMEType != imageMIMEType { updatedImg, err2 := fromImage.UpdatedImage(ctx, types.ManifestUpdateOptions{ ManifestMIMEType: targetMIMEType, }) if err2 != nil { return nil, errors.Wrapf(err, "failed to convert image %q", imageName) } fromImage = updatedImg } config, err := fromImage.ConfigBlob(ctx) if err != nil { return nil, errors.Wrapf(err, "failed to get config from %q", transports.ImageName(fromImage.Reference())) } var imgSpec docker.Image if err := json.Unmarshal(config, &imgSpec); err != nil { return nil, errors.Wrapf(err, "failed to parse config into %s", manifest.DockerV2Schema2MediaType) } return &imgSpec, nil } // UpdateV2Image update the image info depending on the current environment func UpdateV2Image(docker *docker.Image) error { if docker == nil { return nil } if docker.Config != nil { docker.ContainerConfig = *docker.Config } docker.Config = &docker.ContainerConfig if docker.Created.IsZero() { docker.Created = time.Now().UTC() } if docker.OS == "" { docker.OS = runtime.GOOS } if docker.Architecture == "" { docker.Architecture = runtime.GOARCH } if docker.Architecture != runtime.GOARCH { // NOTE:cross-architecture build is not supported currently return errors.Errorf("the architecture does not match, have %q want %q", docker.Architecture, runtime.GOARCH) } if docker.Config.Hostname == "" { docker.Config.Hostname = "isula" } return nil } // ResolveFromImage pull the FROM image and instantiate it func ResolveFromImage(opt *PrepareImageOptions) (types.Image, *storage.Image, error) { ref, si, err := pullAndGetImageInfo(opt) if err != nil { return nil, nil, err } img, err := instantiatingImage(opt.Ctx, opt.SystemContext, ref) if err != nil { return nil, nil, err } return img, si, nil } // GetRWLayerByImageID get the RW layer by image ID func GetRWLayerByImageID(imgID string, store store.Store) (*ContainerDescribe, error) { var ( container *storage.Container err error containerName string ) for { randNum, rerr := util.GenerateCryptoNum(constant.DefaultIDLen) if rerr != nil { return nil, rerr } containerName = fmt.Sprintf("isula-build-%s", randNum) container, err = store.CreateContainer("", []string{containerName}, imgID, "", "", nil) if err == nil { break } if errors.Cause(err) != storage.ErrDuplicateName { return nil, errors.Wrapf(err, "error creating container") } } defer func() { if err != nil { if errRollBack := store.DeleteContainer(container.ID); errRollBack != nil { logrus.Warnf("Failed to deleting container %q in rollback: %v", container.ID, errRollBack) } } }() mountpoint, err := store.Mount(container.ID, "") if err != nil { return nil, errors.Wrapf(err, "error mounting build container %q", container.ID) } return &ContainerDescribe{ ContainerName: containerName, ContainerID: container.ID, Mountpoint: mountpoint, }, nil } // GenerateFromImageSpec generate the image spec func GenerateFromImageSpec(ctx context.Context, fromImage types.Image, targetMIMEType string) (*docker.Image, error) { var ( docker *docker.Image err error ) if fromImage == nil { docker = createScratchV2Image() } else if docker, err = createImageV2Image(ctx, fromImage, targetMIMEType); err != nil { return nil, err } if err = UpdateV2Image(docker); err != nil { return nil, err } return docker, nil } // ResolveImageName resolves the params of image name in FROM command // The image name format can be or : or @ // and it can consists with params such as ${module}_${feature}_${platform}:${version} func ResolveImageName(s string, resolveArg func(string) string) (string, error) { // check special case "\$", so we can better resolve param later newStr := strings.TrimSpace(s) if strings.Contains(newStr, "\\$") { return "", errors.Errorf("image name [%q] is invalid", s) } // convert name newStr, err := dockerfile.ResolveParam(newStr, true, resolveArg) if err != nil { return "", err } logrus.Infof("Input image name is %q, resolved to %q", s, newStr) // validate name if _, err := reference.Parse(newStr); err != nil { return "", err } return newStr, nil } // FindImage get the image from storage by image describe func FindImage(store store.Store, image string) (types.ImageReference, *storage.Image, error) { names, _, err := ResolveName(image, nil, store) if err != nil { return nil, nil, errors.Wrapf(err, "error parsing name %q", image) } var ( ref types.ImageReference img *storage.Image ) for _, name := range names { ref, err = is.Transport.ParseStoreReference(store, name) if err != nil { logrus.Debugf("Error parsing reference to image %q: %v", name, err) continue } img, err = is.Transport.GetStoreImage(store, ref) if err != nil { img2, err2 := store.Image(name) if err2 != nil { logrus.Debugf("Error locating image %q: %v", name, err2) continue } img = img2 } break } if ref == nil || img == nil { return nil, nil, errors.Errorf("locating image %q with name: %v failed", image, names) } return ref, img, nil } // ResolveName checks whether the image name is valid, if the name does not include a domain, // returns a list of candidates it might be func ResolveName(name string, sc *types.SystemContext, store store.Store) ([]string, string, error) { // 1. check name valid if name == "" { return nil, "", nil } // 2. try to find image with name or id in local store if imageID := tryResolveNameInStore(name, store); imageID != "" { return []string{imageID}, "", nil } // 3. try to resolve image name as a transport:destination format dest, transport := tryResolveNameWithTransport(name) if dest != "" && transport != "" { return []string{dest}, transport, nil } // 4. try to resolve image name as a docker reference // if error occurred in this step, there is no need to continue dest, transport, err := tryResolveNameWithDockerReference(name) if err != nil { return nil, "", err } if dest != "" && transport != "" { return []string{dest}, transport, nil } // 5. finally, try to resolve image name in registries candidates, transport := tryResolveNameInRegistries(name, sc) return candidates, transport, nil } func tryResolveNameInStore(name string, store store.Store) string { logrus.Infof("Try to find image: %s in local storage", name) img, err := store.Image(name) if err != nil { return "" } return img.ID } func tryResolveNameWithTransport(name string) (string, string) { logrus.Infof("Try to resolve name: %s with transport", name) splits := strings.SplitN(name, ":", 2) if len(splits) == 2 { if trans := transports.Get(splits[0]); trans != nil { switch trans.Name() { case "docker": // trim prefix if dest like docker://registry.example.com format dest := strings.TrimPrefix(splits[1], "//") return dest, trans.Name() case "docker-archive": dest := strings.TrimSpace(splits[1]) return dest, trans.Name() } } } return "", "" } func tryResolveNameWithDockerReference(name string) (string, string, error) { logrus.Infof("Try to resolve name: %s with docker reference", name) named, err := reference.ParseNormalizedNamed(name) if err != nil { return "", "", errors.Wrapf(err, "error parsing image name %q", name) } if named.String() == name { return name, util.DefaultTransport, nil } domain := reference.Domain(named) if pathPrefix, ok := util.DefaultRegistryPathPrefix[domain]; ok { repoPath := reference.Path(named) tag := "" if tagged, ok := named.(reference.Tagged); ok { tag = ":" + tagged.Tag() } digest := "" if digested, ok := named.(reference.Digested); ok { digest = "@" + digested.Digest().String() } defaultPrefix := pathPrefix + string(os.PathSeparator) if strings.HasPrefix(repoPath, defaultPrefix) && path.Join(domain, repoPath[len(defaultPrefix):])+tag+digest == name { return name, util.DefaultTransport, nil } } return "", "", nil } func tryResolveNameInRegistries(name string, sc *types.SystemContext) ([]string, string) { logrus.Infof("Try to resolve name: %s with in registries", name) var registries []string searchRegistries, err := sysregistriesv2.UnqualifiedSearchRegistries(sc) if err != nil { logrus.Debugf("Unable to read configured registries to complete %q: %v", name, err) searchRegistries = nil } for _, registry := range searchRegistries { reg, err := sysregistriesv2.FindRegistry(sc, registry) if err != nil { logrus.Debugf("Unable to read registry configuration for %#v: %v", registry, err) continue } if reg == nil || !reg.Blocked { registries = append(registries, registry) } } var candidates []string initRegistries := []string{"localhost"} for _, registry := range append(initRegistries, registries...) { if registry == "" { continue } middle := "" if prefix, ok := util.DefaultRegistryPathPrefix[registry]; ok && !strings.ContainsRune(name, '/') { middle = prefix } candidate := path.Join(registry, middle, name) candidates = append(candidates, candidate) } return candidates, util.DefaultTransport }