提交 8d65eab6 编写于 作者: O openeuler-ci-bot 提交者: Gitee

!15 save: add save command for isula-build project

Merge pull request !15 from DCCooper/save
此差异已折叠。
......@@ -40,6 +40,8 @@ service Control {
rpc Import(stream ImportRequest) returns (ImportResponse);
// Tag requests to tag an image
rpc tag(TagRequest) returns (google.protobuf.Empty);
// Save saves the image to tarball
rpc Save(SaveRequest) returns (stream SaveResponse);
}
message BuildRequest {
......@@ -199,3 +201,21 @@ message LoadResponse {
// log is the log sent to client
string log = 1;
}
message SaveRequest {
// image is the image to save
string image = 1;
// path is location for output tarball
string path = 2;
// saveID is the unique ID for each time save
// also is the part to construct tempory path to
// store transport file
string saveID = 3;
}
message SaveResponse {
// data is the tarball of the saved image
bytes data = 1;
// log is log send to cli
string log = 2;
}
......@@ -119,7 +119,7 @@ func NewBuilder(ctx context.Context, store store.Store, req *pb.BuildRequest, ru
defer func(dir string) {
if err != nil {
if rerr := os.RemoveAll(dir); rerr != nil {
logrus.WithField(util.LogKeyBuildID, b.buildID).
logrus.WithField(util.LogKeySessionID, b.buildID).
Warnf("Removing dir in rollback failed: %v", rerr)
}
}
......@@ -176,7 +176,7 @@ func (b *Builder) parseOutput(output string) error {
// Logger adds the "buildID" attribute to build logs
func (b *Builder) Logger() *logrus.Entry {
return logrus.WithField(util.LogKeyBuildID, b.ctx.Value(util.LogFieldKey(util.LogKeyBuildID)))
return logrus.WithField(util.LogKeySessionID, b.ctx.Value(util.LogFieldKey(util.LogKeySessionID)))
}
func (b *Builder) parseBuildArgs(buildArgs []string, key string) (map[string]string, error) {
......
......@@ -219,7 +219,7 @@ func TestCmdBuilderCommit(t *testing.T) {
Raw: "s=1111",
}
ctx := context.WithValue(context.Background(), util.LogFieldKey(util.LogKeyBuildID), "0123456789")
ctx := context.WithValue(context.Background(), util.LogFieldKey(util.LogKeySessionID), "0123456789")
ctx = context.WithValue(ctx, util.BuildDirKey(util.BuildDir), "/tmp/isula-build-test")
s := &stageBuilder{
localStore: localStore,
......
......@@ -87,6 +87,7 @@ func NewContainerImageBuildCmd() *cobra.Command {
NewLoadCmd(),
NewImportCmd(),
NewTagCmd(),
NewSaveCmd(),
)
return ctrImgBuildCmd
......
......@@ -58,6 +58,7 @@ type mockDaemon struct {
loginReq *pb.LoginRequest
logoutReq *pb.LogoutRequest
importReq *pb.ImportRequest
saveReq *pb.SaveRequest
}
func newMockDaemon() *mockDaemon {
......
......@@ -41,6 +41,7 @@ type mockGrpcClient struct {
logoutFunc func(ctx context.Context, in *pb.LogoutRequest, opts ...grpc.CallOption) (*pb.LogoutResponse, error)
loadFunc func(ctx context.Context, in *pb.LoadRequest, opts ...grpc.CallOption) (pb.Control_LoadClient, error)
importFunc func(ctx context.Context, opts ...grpc.CallOption) (pb.Control_ImportClient, error)
saveFunc func(ctx context.Context, in *pb.SaveRequest, opts ...grpc.CallOption) (pb.Control_SaveClient, error)
}
func (gcli *mockGrpcClient) Build(ctx context.Context, in *pb.BuildRequest, opts ...grpc.CallOption) (pb.Control_BuildClient, error) {
......@@ -64,6 +65,13 @@ func (gcli *mockGrpcClient) Remove(ctx context.Context, in *pb.RemoveRequest, op
return &mockRemoveClient{}, nil
}
func (gcli *mockGrpcClient) Save(ctx context.Context, in *pb.SaveRequest, opts ...grpc.CallOption) (pb.Control_SaveClient, error) {
if gcli.saveFunc != nil {
return gcli.saveFunc(ctx, in, opts...)
}
return &mockSaveClient{}, nil
}
func (gcli *mockGrpcClient) List(ctx context.Context, in *pb.ListRequest, opts ...grpc.CallOption) (*pb.ListResponse, error) {
return &pb.ListResponse{
Images: []*pb.ListResponse_ImageInfo{{
......@@ -146,6 +154,10 @@ type mockLoadClient struct {
grpc.ClientStream
}
type mockSaveClient struct {
grpc.ClientStream
}
func (bcli *mockBuildClient) Recv() (*pb.BuildResponse, error) {
resp := &pb.BuildResponse{
ImageID: imageID,
......@@ -189,6 +201,13 @@ func (lcli *mockLoadClient) Recv() (*pb.LoadResponse, error) {
return resp, io.EOF
}
func (scli *mockSaveClient) Recv() (*pb.SaveResponse, error) {
resp := &pb.SaveResponse{
Data: nil,
}
return resp, io.EOF
}
func TestGetStartTimeout(t *testing.T) {
type args struct {
timeout string
......
// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
// iSula-Kits licensed under the Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
// PURPOSE.
// See the Mulan PSL v2 for more details.
// Author: Xiang Li
// Create: 2020-07-31
// Description: This file is used for "save" command
package main
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"github.com/containers/storage/pkg/stringid"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
constant "isula.org/isula-build"
pb "isula.org/isula-build/api/services"
"isula.org/isula-build/exporter"
"isula.org/isula-build/util"
)
type saveOptions struct {
image string
path string
saveID string
}
var saveOpts saveOptions
const (
saveExample = `isula-build ctr-img save busybox:latest -o busybox.tar
isula-build ctr-img save 21c3e96ac411 -o myimage.tar`
)
// NewSaveCmd cmd for container image saving
func NewSaveCmd() *cobra.Command {
saveCmd := &cobra.Command{
Use: "save",
Short: "Save image to tarball",
Example: saveExample,
RunE: saveCommand,
}
saveCmd.PersistentFlags().StringVarP(&saveOpts.path, "output", "o", "", "Path to save the tarball")
return saveCmd
}
func saveCommand(cmd *cobra.Command, args []string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cli, err := NewClient(ctx)
if err != nil {
return err
}
return runSave(ctx, cli, args)
}
func runSave(ctx context.Context, cli Cli, args []string) error {
if len(args) != 1 {
return errors.New("save accepts only one image")
}
if len(saveOpts.path) == 0 {
return errors.New("output path should not be empty")
}
saveOpts.saveID = stringid.GenerateNonCryptoID()[:constant.DefaultIDLen]
if !filepath.IsAbs(saveOpts.path) {
pwd, err := os.Getwd()
if err != nil {
return errors.New("get current path failed")
}
saveOpts.path = util.MakeAbsolute(saveOpts.path, pwd)
}
saveOpts.image = args[0]
saveStream, err := cli.Client().Save(ctx, &pb.SaveRequest{
Image: saveOpts.image,
Path: saveOpts.path,
SaveID: saveOpts.saveID,
})
if err != nil {
return err
}
fileChan := make(chan []byte, constant.BufferSize)
eg, _ := errgroup.WithContext(ctx)
eg.Go(func() error {
defer close(fileChan)
for {
msg, err := saveStream.Recv()
if err == io.EOF {
break
}
if err != nil {
return err
}
fileChan <- msg.Data
fmt.Print(msg.Log)
}
return nil
})
eg.Go(func() error {
if err := exporter.ArchiveRecv(ctx, saveOpts.path, false, fileChan); err != nil {
return err
}
return nil
})
if err := eg.Wait(); err != nil {
if rErr := os.Remove(saveOpts.path); rErr != nil {
logrus.Warnf("Removing save output tarball %q failed: %v", saveOpts.path, rErr)
}
return errors.Errorf("save image failed: %v", err)
}
fmt.Printf("Save success with image: %s\n", saveOpts.image)
return nil
}
......@@ -14,8 +14,11 @@
package daemon
import (
"bufio"
"context"
"io"
"io/ioutil"
"os"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
......@@ -33,7 +36,7 @@ func (b *Backend) Build(req *pb.BuildRequest, stream pb.Control_BuildServer) (er
"BuildID": req.GetBuildID(),
}).Info("BuildRequest received")
ctx := context.WithValue(stream.Context(), util.LogFieldKey(util.LogKeyBuildID), req.BuildID)
ctx := context.WithValue(stream.Context(), util.LogFieldKey(util.LogKeySessionID), req.BuildID)
builder, err := b.daemon.NewBuilder(ctx, req)
if err != nil {
return err
......@@ -48,11 +51,12 @@ func (b *Backend) Build(req *pb.BuildRequest, stream pb.Control_BuildServer) (er
}()
var (
f *os.File
length int
imageID string
pipeFile string
eg *errgroup.Group
fileChan chan []byte
errc = make(chan error, 1)
errC = make(chan error, 1)
)
pipeWrapper := builder.OutputPipeWrapper()
......@@ -71,7 +75,7 @@ func (b *Backend) Build(req *pb.BuildRequest, stream pb.Control_BuildServer) (er
// message into the pipe to make the goroutine move on instead of hangs.
if err != nil && pipeFile != "" {
if perr := ioutil.WriteFile(pipeFile, []byte(err.Error()), constant.DefaultRootFileMode); perr != nil {
logrus.WithField(util.LogKeyBuildID, req.BuildID).Warnf("Write error [%v] in to pipe file failed: %v", err, perr)
logrus.WithField(util.LogKeySessionID, req.BuildID).Warnf("Write error [%v] in to pipe file failed: %v", err, perr)
}
}
......@@ -82,28 +86,43 @@ func (b *Backend) Build(req *pb.BuildRequest, stream pb.Control_BuildServer) (er
if pipeWrapper == nil {
return nil
}
fileChan, err = exporter.PipeArchiveStream(req.BuildID, pipeWrapper)
f, err = exporter.PipeArchiveStream(pipeWrapper)
defer func() {
if cErr := f.Close(); cErr != nil {
logrus.WithField(util.LogKeySessionID, req.BuildID).Warnf("Closing archive stream pipe %q failed: %v", pipeWrapper.PipeFile, cErr)
}
}()
if err != nil {
return err
}
for c := range fileChan {
reader := bufio.NewReader(f)
buf := make([]byte, constant.BufferSize, constant.BufferSize)
for {
length, err = reader.Read(buf)
if length == 0 && pipeWrapper.Done {
break
}
if err != nil && err != io.EOF {
return err
}
if err = stream.Send(&pb.BuildResponse{
Data: c,
Data: buf[0:length],
}); err != nil {
return err
}
}
return pipeWrapper.Err
logrus.WithField(util.LogKeySessionID, req.BuildID).Debugf("Piping build archive stream done")
return nil
})
go func() {
errc <- eg.Wait()
errC <- eg.Wait()
}()
select {
case err = <-errc:
close(errc)
case err = <-errC:
close(errC)
if err != nil {
return err
}
......@@ -117,7 +136,7 @@ func (b *Backend) Build(req *pb.BuildRequest, stream pb.Control_BuildServer) (er
case <-stream.Context().Done():
err = ctx.Err()
if err != nil && err != context.Canceled {
logrus.WithField(util.LogKeyBuildID, req.BuildID).Warnf("Stream closed with: %v", err)
logrus.WithField(util.LogKeySessionID, req.BuildID).Warnf("Stream closed with: %v", err)
}
}
......
// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
// iSula-Kits licensed under the Mulan PSL v2.
// You can use this software according to the terms and conditions of the Mulan PSL v2.
// You may obtain a copy of Mulan PSL v2 at:
// http://license.coscl.org.cn/MulanPSL2
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
// PURPOSE.
// See the Mulan PSL v2 for more details.
// Author: Xiang Li
// Create: 2020-07-31
// Description: This file is "save" command for backend
package daemon
import (
"bufio"
"context"
"fmt"
"io"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
constant "isula.org/isula-build"
pb "isula.org/isula-build/api/services"
"isula.org/isula-build/exporter"
"isula.org/isula-build/image"
"isula.org/isula-build/pkg/logger"
"isula.org/isula-build/util"
)
// Save receives a save request and save the image into tarball
func (b *Backend) Save(req *pb.SaveRequest, stream pb.Control_SaveServer) (err error) { // nolint:gocyclo
const exportType = "docker-archive"
var (
f *os.File
length int
pipeWrapper *exporter.PipeWrapper
imageInfo = req.Image
saveID = req.SaveID
store = b.daemon.localStore
errC = make(chan error, 1)
runDir = filepath.Join(b.daemon.opts.RunRoot, "save", saveID)
cliLogger = logger.NewCliLogger(constant.CliLogBufferLen)
)
logrus.WithFields(logrus.Fields{
"SaveID": req.GetSaveID(),
}).Info("SaveRequest received")
if err = os.MkdirAll(runDir, constant.DefaultRootDirMode); err != nil {
return err
}
defer func() {
if rErr := os.RemoveAll(runDir); rErr != nil {
logrus.Errorf("Remove saving dir %q failed: %v", runDir, rErr)
err = rErr
}
}()
imageID, err := store.Lookup(imageInfo)
if err != nil {
logrus.Errorf("Lookup image %s failed: %v", imageInfo, err)
return err
}
pipeWrapper, err = exporter.NewPipeWrapper(runDir, exportType)
if err != nil {
return err
}
ctx := context.WithValue(stream.Context(), util.LogFieldKey(util.LogKeySessionID), saveID)
eg, egCtx := errgroup.WithContext(ctx)
eg.Go(func() error {
defer func() {
pipeWrapper.Close()
cliLogger.CloseContent()
}()
output := fmt.Sprintf("%s:%s", exportType, pipeWrapper.PipeFile)
exOpts := exporter.ExportOptions{
SystemContext: image.GetSystemContext(),
Ctx: ctx,
ReportWriter: cliLogger,
}
if err = exporter.Export(imageID, output, exOpts, store); err != nil {
logrus.Errorf("Save image %s failed: %v", imageID, err)
return err
}
return nil
})
eg.Go(func() error {
f, err = exporter.PipeArchiveStream(pipeWrapper)
if err != nil {
return err
}
defer func() {
if cErr := f.Close(); cErr != nil {
logrus.WithField(util.LogKeySessionID, req.SaveID).Warnf("Closing save archive stream pipe %q failed: %v", pipeWrapper.PipeFile, cErr)
}
}()
reader := bufio.NewReader(f)
buf := make([]byte, constant.BufferSize, constant.BufferSize)
for {
length, err = reader.Read(buf)
if err == io.EOF {
break
}
if err != nil {
return err
}
if err = stream.Send(&pb.SaveResponse{
Data: buf[0:length],
}); err != nil {
return err
}
}
logrus.WithField(util.LogKeySessionID, req.SaveID).Debugf("Piping save archive stream done")
return nil
})
eg.Go(func() error {
for content := range cliLogger.GetContent() {
if content == "" {
return nil
}
if err = stream.Send(&pb.SaveResponse{
Log: content,
}); err != nil {
return err
}
}
return nil
})
go func() {
errC <- eg.Wait()
}()
select {
case err = <-errC:
close(errC)
if err != nil {
return err
}
// export done in another go routine, so send nil data
if err = stream.Send(&pb.SaveResponse{Data: nil}); err != nil {
return err
}
case <-stream.Context().Done():
err = egCtx.Err()
if err != nil && err != context.Canceled {
logrus.WithField(util.LogKeySessionID, saveID).Warnf("Stream closed with: %v", err)
}
}
return nil
}
......@@ -13,8 +13,10 @@
* [--iidfile](#--iidfile)
* [-o, --output](#-o---output)
* [--proxy](#--proxy)
* [--tag](#--tag)
* [Viewing a Local Persistent Image](#viewing-a-local-persistent-image)
* [Importing a Base Image from a Tarball](#importing-a-base-image-from-a-tarball)
* [Saving a Local Persistent Image](#saving-a-local-persistent-image)
* [Deleting a Local Persistent Image](#deleting-a-local-persistent-image)
* [-a, --all](#-a---all)
* [-p, --prune](#-p---prune)
......@@ -233,15 +235,33 @@ Usage:
`isula-build ctr-img import file [REPOSITORY[:TAG]]`
```bash
$ sudo isula-build ctr-img busybox.tar
$ sudo isula-build ctr-img import busybox.tar
Import success with image id: bf7b3b8ad6d842fb6e0c2dd60727ccb60a86c0e8781a35ae39de5aeef9979189
```
```bash
$ sudo isula-build ctr-img busybox.tar busybox:isula
$ sudo isula-build ctr-img import busybox.tar busybox:isula
Import success with image id: 2d77083e646bf77e25547ea489b00ed8ec318cc37ba81c41e7ec92bca2845033
```
### Saving a Local Persistent Image
we can run the `save` command to save the image stored locally and make it a tarball.
Usage:
`isula-build ctr-img save [REPOSITORY:TAG]|imageID -o xx.tar`
```bash
$ sudo isula-build ctr-img save busybox:latest -o busybox.tar
Save success with image: busybox:latest
```
```bash
$ sudo isula-build ctr-img save 21c3e96ac411 -o busybox.tar
Save success with image: 21c3e96ac411
```
### Deleting a Local Persistent Image
We can run the `rm` command to delete the image stored locally.
......
......@@ -57,7 +57,7 @@ type ExportOptions struct {
// Export export an archive to the client
func Export(src, destSpec string, iopts ExportOptions, localStore store.Store) error {
eLog := logrus.WithField(util.LogKeyBuildID, iopts.Ctx.Value(util.LogFieldKey(util.LogKeyBuildID)))
eLog := logrus.WithField(util.LogKeySessionID, iopts.Ctx.Value(util.LogFieldKey(util.LogKeySessionID)))
if destSpec == "" {
return nil
}
......@@ -198,11 +198,8 @@ func NewPipeWrapper(runDir, expt string) (*PipeWrapper, error) {
}
// PipeArchiveStream pipes the GRPC stream with pipeFile
func PipeArchiveStream(buildID string, pipeWrapper *PipeWrapper) (fc chan []byte, err error) {
var (
file *os.File
length int
)
func PipeArchiveStream(pipeWrapper *PipeWrapper) (f *os.File, err error) {
var file *os.File
if pipeWrapper == nil || pipeWrapper.PipeFile == "" {
return nil, errors.New("no pipe wrapper found")
}
......@@ -210,32 +207,7 @@ func PipeArchiveStream(buildID string, pipeWrapper *PipeWrapper) (fc chan []byte
if file, err = os.OpenFile(pipeWrapper.PipeFile, os.O_RDONLY, os.ModeNamedPipe); err != nil {
return nil, err
}
reader := bufio.NewReader(file)
buf := make([]byte, constant.BufferSize, constant.BufferSize)
fc = make(chan []byte, constant.BufferSize)
go func() {
defer func() {
if cerr := file.Close(); cerr != nil {
logrus.WithField(util.LogKeyBuildID, buildID).Warnf("Closing archive stream pipe %q failed: %v", pipeWrapper.PipeFile, cerr)
}
close(fc)
}()
for {
if length, err = reader.Read(buf); err != nil && err != io.EOF {
pipeWrapper.Err = err
}
bytes := make([]byte, length, length)
copy(bytes, buf[0:length])
fc <- bytes
if length == 0 && pipeWrapper.Done {
break
}
}
}()
logrus.WithField(util.LogKeyBuildID, buildID).Debugf("Piping archive stream done")
return fc, nil
return file, nil
}
// ArchiveRecv receive data stream and write to file
......
......@@ -100,7 +100,7 @@ type pullOption struct {
}
func pullImage(opt pullOption) (types.ImageReference, error) {
pLog := logrus.WithField(util.LogKeyBuildID, opt.ctx.Value(util.LogFieldKey(util.LogKeyBuildID)))
pLog := logrus.WithField(util.LogKeySessionID, opt.ctx.Value(util.LogFieldKey(util.LogKeySessionID)))
policy, err := signature.DefaultPolicy(opt.sc)
if err != nil {
return nil, errors.Wrapf(err, "error obtaining default signature policy")
......@@ -131,7 +131,7 @@ func pullImage(opt pullOption) (types.ImageReference, error) {
}
func pullAndGetImageInfo(opt *PrepareImageOptions) (types.ImageReference, *storage.Image, error) {
pLog := logrus.WithField(util.LogKeyBuildID, opt.Ctx.Value(util.LogFieldKey(util.LogKeyBuildID)))
pLog := logrus.WithField(util.LogKeySessionID, opt.Ctx.Value(util.LogFieldKey(util.LogKeySessionID)))
candidates, transport, err := ResolveName(opt.FromImage, opt.SystemContext, opt.Store)
if err != nil {
return nil, nil, errors.Wrapf(err, "error parsing reference to image %q", opt.FromImage)
......
......@@ -125,7 +125,7 @@ func (r *OCIRunner) Run() error {
}
func (r *OCIRunner) runContainer() (wstatus unix.WaitStatus, err error) { // nolint:gocyclo
pLog := logrus.WithField(util.LogKeyBuildID, r.ctx.Value(util.LogFieldKey(util.LogKeyBuildID)))
pLog := logrus.WithField(util.LogKeySessionID, r.ctx.Value(util.LogFieldKey(util.LogKeySessionID)))
containerName := filepath.Base(r.bundlePath)
pidFile := filepath.Join(r.bundlePath, "pid")
createOpts := runc.CreateOpts{
......
......@@ -49,8 +49,8 @@ const (
// DefaultTransport is default transport
DefaultTransport = "docker://"
// LogKeyBuildID describes the key field with buildID for logrus
LogKeyBuildID = "buildID"
// LogKeySessionID describes the key field with sessionID for logrus
LogKeySessionID = "sessionID"
// BuildDir describes the key field with BuildDir in build context
BuildDir = "buildDir"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册