import: support importing base image from a tarball

Signed-off: liuzekun <liuzekun@huawei.com>

api/services/control.proto

@@ -36,6 +36,8 @@ service Control {
     rpc Logout(LogoutRequest) returns (LogoutResponse);
     // Load requests an image tar load
     rpc Load(LoadRequest) returns (LoadResponse);
+    // Import requests import a new image
+    rpc Import(stream ImportRequest) returns (ImportResponse);
 }
 
 message BuildRequest {
@@ -61,6 +63,18 @@ message BuildRequest {
     string encryptKey = 10;
 }
 
+message ImportRequest {
+    // reference is reference of the import image
+    string reference = 1;
+    // data is the tarball of the import image
+    bytes data = 2;
+}
+
+message ImportResponse {
+    // imageID is the ID of the import image
+    string imageID = 1;
+}
+
 message BuildStatic {
     // buildTime is a fixed time for binary equivalence build
     google.protobuf.Timestamp buildTime = 1;

builder/dockerfile/cmd_builder_commit.go

@@ -40,7 +40,8 @@ func newImageCopyOptions(reportWriter io.Writer) *cp.Options {
 	}
 }
 
-func getPolicyContext() (*signature.PolicyContext, error) {
+// GetPolicyContext returns a specied policy context
+func GetPolicyContext() (*signature.PolicyContext, error) {
 	systemContext := image.GetSystemContext()
 	systemContext.DirForceCompress = true
 	commitPolicy, err := signature.DefaultPolicy(systemContext)
@@ -124,7 +125,7 @@ func (c *cmdBuilder) commit(ctx context.Context) (string, error) {
 		exporting = !c.isFromImageExist(storeTransport)
 	}
 
-	policyContext, err := getPolicyContext()
+	policyContext, err := GetPolicyContext()
 	if err != nil {
 		return "", err
 	}

cmd/cli/build.go

@@ -84,6 +84,7 @@ func NewContainerImageBuildCmd() *cobra.Command {
 		NewImagesCmd(),
 		NewRemoveCmd(),
 		NewLoadCmd(),
+		NewImportCmd(),
 	)
 
 	return ctrImgBuildCmd

cmd/cli/build_test.go

@@ -57,12 +57,17 @@ type mockDaemon struct {
 	loadReq   *pb.LoadRequest
 	loginReq  *pb.LoginRequest
 	logoutReq *pb.LogoutRequest
+	importReq *pb.ImportRequest
 }
 
 func newMockDaemon() *mockDaemon {
 	return &mockDaemon{}
 }
 
+func (f *mockDaemon) importImage(_ context.Context, opts ...grpc.CallOption) (pb.Control_ImportClient, error) {
+	return &mockImportClient{}, nil
+}
+
 func (f *mockDaemon) build(_ context.Context, in *pb.BuildRequest, opts ...grpc.CallOption) (pb.Control_BuildClient, error) {
 	f.buildReq = in
 	return &mockBuildClient{}, nil

cmd/cli/grpc_client_test.go

@@ -40,6 +40,7 @@ type mockGrpcClient struct {
 	loginFunc       func(ctx context.Context, in *pb.LoginRequest, opts ...grpc.CallOption) (*pb.LoginResponse, error)
 	logoutFunc      func(ctx context.Context, in *pb.LogoutRequest, opts ...grpc.CallOption) (*pb.LogoutResponse, error)
 	loadFunc        func(ctx context.Context, in *pb.LoadRequest, opts ...grpc.CallOption) (*pb.LoadResponse, error)
+	importFunc      func(ctx context.Context, opts ...grpc.CallOption) (pb.Control_ImportClient, error)
 }
 
 func (gcli *mockGrpcClient) Build(ctx context.Context, in *pb.BuildRequest, opts ...grpc.CallOption) (pb.Control_BuildClient, error) {
@@ -49,6 +50,13 @@ func (gcli *mockGrpcClient) Build(ctx context.Context, in *pb.BuildRequest, opts
 	return &mockBuildClient{isArchive: true}, nil
 }
 
+func (gcli *mockGrpcClient) Import(ctx context.Context, opts ...grpc.CallOption) (pb.Control_ImportClient, error) {
+	if gcli.importFunc != nil {
+		return gcli.importFunc(ctx, opts...)
+	}
+	return nil, nil
+}
+
 func (gcli *mockGrpcClient) Remove(ctx context.Context, in *pb.RemoveRequest, opts ...grpc.CallOption) (pb.Control_RemoveClient, error) {
 	if gcli.removeFunc != nil {
 		return gcli.removeFunc(ctx, in, opts...)
@@ -118,6 +126,10 @@ type mockBuildClient struct {
 	isArchive bool
 }
 
+type mockImportClient struct {
+	grpc.ClientStream
+}
+
 type mockStatusClient struct {
 	grpc.ClientStream
 }
@@ -137,6 +149,17 @@ func (bcli *mockBuildClient) Recv() (*pb.BuildResponse, error) {
 	return resp, nil
 }
 
+func (icli *mockImportClient) CloseAndRecv() (*pb.ImportResponse, error) {
+	resp := &pb.ImportResponse{
+		ImageID: imageID,
+	}
+	return resp, nil
+}
+
+func (icli *mockImportClient) Send(*pb.ImportRequest) error {
+	return nil
+}
+
 func (scli *mockStatusClient) Recv() (*pb.StatusResponse, error) {
 	resp := &pb.StatusResponse{
 		Content: content,

cmd/cli/import.go

+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//     http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Zekun Liu
+// Create: 2020-07-16
+// Description: This file is used for command import
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	dockerref "github.com/containers/image/v5/docker/reference"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	pb "isula.org/isula-build/api/services"
+	"isula.org/isula-build/util"
+)
+
+const (
+	bufferSize     = 32 * 1024
+	maxTarballSize = 10 * 1024 * 1024 * 1024 // support tarball max size at most 10G
+	importExample  = `isula-build ctr-img import file [REPOSITORY[:TAG]]`
+	importArgsLen  = 1
+)
+
+type importOptions struct {
+	source    string
+	reference string
+}
+
+var importOpts importOptions
+
+// NewImportCmd returns import command
+func NewImportCmd() *cobra.Command {
+	importCmd := &cobra.Command{
+		Use:     "import",
+		Short:   "Import the base image from a tarball to the image store",
+		Example: importExample,
+		RunE:    importCommand,
+	}
+	return importCmd
+}
+
+func importCommand(c *cobra.Command, args []string) error {
+	if len(args) < importArgsLen {
+		return errors.New("requires at least one argument")
+	}
+	if err := util.CheckFileSize(args[0], maxTarballSize); err != nil {
+		return err
+	}
+	importOpts.source = args[0]
+	if len(args) > importArgsLen {
+		importOpts.reference = args[1]
+	}
+
+	ctx := context.TODO()
+	cli, err := NewClient(ctx)
+	if err != nil {
+		return err
+	}
+	return runImport(ctx, cli)
+}
+
+func runImport(ctx context.Context, cli Cli) error {
+	if importOpts.reference != "" {
+		if _, err := dockerref.Parse(importOpts.reference); err != nil {
+			return err
+		}
+	}
+	file, err := os.Open(importOpts.source)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if ferr := file.Close(); ferr != nil {
+			logrus.Warnf("Close file %s failed", importOpts.source)
+		}
+	}()
+
+	rpcCli, err := cli.Client().Import(ctx)
+	if err != nil {
+		return err
+	}
+
+	reader := bufio.NewReader(file)
+	buf := make([]byte, bufferSize, bufferSize)
+	var length int
+	for {
+		length, err = reader.Read(buf)
+		if err != nil && err != io.EOF {
+			return err
+		}
+		if length == 0 {
+			break
+		}
+		if err = rpcCli.Send(&pb.ImportRequest{
+			Data:      buf[0:length],
+			Reference: importOpts.reference,
+		}); err != nil {
+			return err
+		}
+	}
+
+	resp, err := rpcCli.CloseAndRecv()
+	if err != nil {
+		return err
+	}
+	if resp == nil {
+		return errors.New("import failed, got nil response")
+	}
+	fmt.Printf("Import success with image id: %s\n", resp.ImageID)
+	return nil
+}

daemon/import.go

+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//     http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Zekun Liu
+// Create: 2020-07-22
+// Description: This file is "import" command for backend
+
+package daemon
+
+import (
+	"io"
+	"strings"
+
+	cp "github.com/containers/image/v5/copy"
+	dockerref "github.com/containers/image/v5/docker/reference"
+	is "github.com/containers/image/v5/storage"
+	"github.com/containers/image/v5/tarball"
+	"github.com/containers/image/v5/transports"
+	"github.com/containers/storage/pkg/stringid"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	pb "isula.org/isula-build/api/services"
+	"isula.org/isula-build/builder/dockerfile"
+	"isula.org/isula-build/util"
+)
+
+const (
+	tmpFilePattern = "isula-build-ctr-img-import"
+	noneReference  = "<none>:<none>"
+	bufLen         = 1024
+)
+
+// Import an image from a tarball
+func (b *Backend) Import(serv pb.Control_ImportServer) error {
+	localStore := b.daemon.localStore
+	buf := make([]byte, 0, bufLen)
+	reference := ""
+	for {
+		msg, ierr := serv.Recv()
+		if ierr == io.EOF {
+			break
+		}
+		if ierr != nil {
+			return ierr
+		}
+		if msg == nil {
+			return errors.New("import failed, receive nil msg")
+		}
+		reference = msg.Reference
+		buf = append(buf, msg.Data...)
+	}
+
+	logrus.Infof("Received and import image %q", reference)
+	reference, err := parseReference(reference)
+	if err != nil {
+		return err
+	}
+	srcRef, err := tarball.NewReference([]string{"-"}, buf)
+	if err != nil {
+		return err
+	}
+
+	tmpName := stringid.GenerateRandomID() + "-import-tmp"
+	dstRef, err := is.Transport.ParseStoreReference(localStore, tmpName)
+	if err != nil {
+		return err
+	}
+	policyContext, err := dockerfile.GetPolicyContext()
+	if err != nil {
+		return err
+	}
+	if _, err = cp.Image(serv.Context(), policyContext, dstRef, srcRef, nil); err != nil {
+		return err
+	}
+	img, err := is.Transport.GetStoreImage(localStore, dstRef)
+	if err != nil {
+		return errors.Wrapf(err, "error locating image %q in local storage after import", transports.ImageName(dstRef))
+	}
+	img.Names = append(img.Names, reference)
+	newNames := util.CopyStringsWithoutSpecificElem(img.Names, tmpName)
+	if err = localStore.SetNames(img.ID, newNames); err != nil {
+		return errors.Wrapf(err, "failed to prune temporary name from image %q", img.ID)
+	}
+
+	resp := &pb.ImportResponse{ImageID: img.ID}
+	if err = serv.SendAndClose(resp); err != nil {
+		return err
+	}
+	logrus.Infof("Import success with image id %q", img.ID)
+
+	return nil
+}
+
+func parseReference(ref string) (string, error) {
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return noneReference, nil
+	}
+	if _, err := dockerref.Parse(ref); err != nil {
+		return "", err
+	}
+
+	return ref, nil
+}

daemon/import_test.go

+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
+// isula-build licensed under the Mulan PSL v2.
+// You can use this software according to the terms and conditions of the Mulan PSL v2.
+// You may obtain a copy of Mulan PSL v2 at:
+//     http://license.coscl.org.cn/MulanPSL2
+// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+// PURPOSE.
+// See the Mulan PSL v2 for more details.
+// Author: Zekun Liu
+// Create: 2020-07-25
+// Description: This is test file for daemon import.go
+
+package daemon
+
+import (
+	"testing"
+
+	"gotest.tools/assert"
+)
+
+func TestParseReference(t *testing.T) {
+	type testcase struct {
+		name      string
+		reference string
+		expect    string
+		isErr     bool
+		errStr    string
+	}
+	var testcases = []testcase{
+		{
+			name:      "repo only",
+			reference: "busybox",
+			expect:    "busybox",
+		},
+		{
+			name:      "repo and tag",
+			reference: "busybox:latest",
+			expect:    "busybox:latest",
+		},
+		{
+			name:      "ref with tag missing",
+			reference: "busybox:",
+			isErr:     true,
+			errStr:    "invalid reference format",
+		},
+		{
+			name:      "empty ref",
+			reference: "",
+			expect:    noneReference,
+		},
+		{
+			name:      "ref with no tag",
+			reference: "busybox",
+			expect:    "busybox",
+		},
+		{
+			name:      "ref with space",
+			reference: "busybox: latest",
+			isErr:     true,
+			errStr:    "invalid reference format",
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			ref, err := parseReference(tc.reference)
+			assert.Equal(t, err != nil, tc.isErr, tc.name)
+			if err != nil {
+				assert.ErrorContains(t, err, tc.errStr, tc.name)
+			}
+			if err == nil {
+				assert.Equal(t, ref, tc.expect, tc.name)
+			}
+		})
+	}
+}