replace the occlum SDK container with the occlum rpm on host to build the enclave.so

shim/runtime/carrier/constants/constants.go

@@ -91,7 +91,7 @@ start $@`
 	CarrierScript = `#!/bin/bash
 set -xe
 base_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
-occlum_workspace=/occlum_workspace
+occlum_workspace=${base_dir}/occlum_workspace
 
 temp=$(getopt -a -o a:r:w:p:c:e:u:m:s:k:n: -l action:,rootfs:,work_dir:,entry_point:,occlum_config_path:,enclave_config_path:,\
 unsigned_encalve_path:,unsigned_material_path:,signed_enclave_path:,public_key_path:,signature_path: -- "$@")
@@ -151,7 +151,7 @@ function buildUnsignedEnclave(){
     echo "BuildUnsignedEnclave:: the argumentes should not be empty: entry_point, rootfs, work_dir"
     exit 1
   fi
-
+  export PATH=$PATH:/opt/occlum/build/bin/
   rm -fr ${occlum_workspace}
   mkdir -p ${occlum_workspace}
   pushd ${occlum_workspace}

shim/runtime/carrier/occlum/occlum.go

@@ -5,11 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"path/filepath"
-	"strconv"
-	"syscall"
 	"time"
 
 	"github.com/BurntSushi/toml"
@@ -22,8 +19,6 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
 	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/namespaces"
-	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
@@ -31,11 +26,11 @@ import (
 
 const (
 	defaultNamespace         = "k8s.io"
+	startScriptFileName      = "start.sh"
 	replaceOcclumImageScript = "replace_occlum_image.sh"
 	carrierScriptFileName    = "carrier.sh"
-	startScriptFileName      = "start.sh"
 	rootfsDirName            = "rootfs"
-	enclaveDataDir           = "data"
+	dataDirName              = "data"
 )
 
 var _ carrier.Carrier = &occlum{}
@@ -87,144 +82,50 @@ func (c *occlum) BuildUnsignedEnclave(req *task.CreateTaskRequest, args *carrier
 		return "", err
 	}
 
-	namespace, ok := namespaces.Namespace(c.context)
-	if !ok {
-		namespace = defaultNamespace
-	}
-	// Create a new client connected to the default socket path for containerd.
-	client, err := containerd.New(c.shimConfig.Containerd.Socket)
-	if err != nil {
-		return "", fmt.Errorf("failed to create containerd client. error: %++v", err)
-	} else {
-		c.task.client = client
-	}
-	logrus.Debugf("BuildUnsignedEnclave: create containerd client time cost: %d", (time.Now().Sub(timeStart))/time.Second)
-	logrus.Debugf("BuildUnsignedEnclave: get containerd client successfully")
-
-	if err = createNamespaceIfNotExist(client, namespace); err != nil {
-		logrus.Errorf("BuildUnsignedEnclave: create namespace %s failed. error: %++v", namespace, err)
-		return "", err
-	}
-
-	// pull the image that used to build enclave.
-	occlumEnclaveBuilderImage := c.shimConfig.EnclaveRuntime.Occlum.BuildImage
-	timeStart = time.Now()
-	image, err := client.Pull(c.context, occlumEnclaveBuilderImage, containerd.WithPullUnpack)
-	if err != nil {
-		return "", fmt.Errorf("failed to pull image %s. error: %++v", occlumEnclaveBuilderImage, err)
-	}
-	logrus.Debugf("BuildUnsignedEnclave: pull occlum SDK image time cost: %d", (time.Now().Sub(timeStart))/time.Second)
-	logrus.Debugf("BuildUnsignedEnclave: pull image %s successfully", occlumEnclaveBuilderImage)
-
-	// Generate the containerId and snapshotId.
-	// FIXME The variables containerId and snapshotId should be generated by utils.GenerateID
-	rand.Seed(time.Now().UnixNano())
-	containerId := fmt.Sprintf("occlum-enclave-builder-%s", strconv.FormatInt(rand.Int63(), 16))
-	snapshotId := fmt.Sprintf("occlum-enclave-builder-snapshot-%s", strconv.FormatInt(rand.Int63(), 16))
+	// Copy the script files that are used to build encalve.so by occlum into rootfs
+	rootfsDir := filepath.Join(req.Bundle, rootfsDirName)
+	dataDir := filepath.Join(req.Bundle, dataDirName)
+	os.MkdirAll(dataDir, 0755)
 
-	logrus.Debugf("BuildUnsignedEnclave: containerId: %s, snapshotId: %s", containerId, snapshotId)
-
-	if err := os.Mkdir(filepath.Join(req.Bundle, enclaveDataDir), 0755); err != nil {
-		return "", err
-	}
-
-	replaceImagesScript := filepath.Join(req.Bundle, enclaveDataDir, replaceOcclumImageScript)
+	replaceImagesScript := filepath.Join(dataDir, replaceOcclumImageScript)
 	if err := ioutil.WriteFile(replaceImagesScript, []byte(carr_const.ReplaceOcclumImageScript), os.ModePerm); err != nil {
 		return "", err
 	}
 
-	carrierScript := filepath.Join(req.Bundle, enclaveDataDir, carrierScriptFileName)
+	carrierScript := filepath.Join(dataDir, carrierScriptFileName)
 	if err := ioutil.WriteFile(carrierScript, []byte(carr_const.CarrierScript), os.ModePerm); err != nil {
 		return "", err
 	}
 
-	startScript := filepath.Join(req.Bundle, enclaveDataDir, startScriptFileName)
-	if err := ioutil.WriteFile(startScript, []byte(carr_const.StartScript), os.ModePerm); err != nil {
-		return "", err
-	}
-
-	// Create rootfs mount points.
-	mounts := make([]specs.Mount, 0)
-	rootfsMount := specs.Mount{
-		Destination: filepath.Join("/", rootfsDirName),
-		Type:        "bind",
-		Source:      filepath.Join(req.Bundle, rootfsDirName),
-		Options:     []string{"rbind", "rw"},
-	}
-	dataMount := specs.Mount{
-		Destination: filepath.Join("/", enclaveDataDir),
-		Type:        "bind",
-		Source:      filepath.Join(req.Bundle, enclaveDataDir),
-		Options:     []string{"rbind", "rw"},
-	}
-
-	logrus.Debugf("BuildUnsignedEnclave: rootfsMount source: %s, destination: %s",
-		rootfsMount.Source, rootfsMount.Destination)
-
-	mounts = append(mounts, rootfsMount, dataMount)
-	// create a container
-	timeStart = time.Now()
-	container, err := client.NewContainer(
-		c.context,
-		containerId,
-		containerd.WithImage(image),
-		containerd.WithNewSnapshot(snapshotId, image),
-		containerd.WithNewSpec(oci.WithImageConfig(image),
-			oci.WithProcessArgs("/bin/bash", filepath.Join("/", enclaveDataDir, startScriptFileName)),
-			oci.WithPrivileged,
-			oci.WithMounts(mounts),
-		),
-	)
-	if err != nil {
-		return "", fmt.Errorf("failed to create container by image %s. error: %++v",
-			occlumEnclaveBuilderImage, err)
-	} else {
-		c.task.container = &container
-	}
-	logrus.Debugf("BuildUnsignedEnclave: create occlum SDK container time cost: %d", (time.Now().Sub(timeStart))/time.Second)
-
-	// Create a task from the container.
-	t, err := container.NewTask(c.context, cio.NewCreator(cio.WithStdio))
-	if err != nil {
-		return "", err
-	} else {
-		c.task.task = &t
-	}
-	logrus.Debugf("BuildUnsignedEnclave: create task successfully")
-
-	if err := t.Start(c.context); err != nil {
-		logrus.Errorf("BuildUnsignedEnclave: start task failed. error: %++v", err)
-		return "", err
-	}
-
-	cmd := []string{
-		"/bin/bash", filepath.Join("/", enclaveDataDir, carrierScriptFileName),
+	// Execute the carrier script to generate the unsigned enclave.so in rootfs
+	cmdArgs := []string{
+		"/bin/bash", filepath.Join(dataDir, carrierScriptFileName),
 		"--action", "buildUnsignedEnclave",
 		"--entry_point", c.entryPoints[0],
 		"--work_dir", c.workDirectory,
-		"--rootfs", filepath.Join("/", rootfsDirName),
+		"--rootfs", rootfsDir,
 	}
 	var occlumConfigPath string
 	if c.configPath != "" {
-		occlumConfigPath = filepath.Join("/", rootfsDirName, c.configPath)
+		occlumConfigPath = filepath.Join(rootfsDir, c.configPath)
 	} else {
 		c.configPath = "Occlum.json"
-		occlumConfigPath = filepath.Join("/", enclaveDataDir, c.configPath)
-		hostPath := filepath.Join(c.bundle, enclaveDataDir, c.configPath)
-		if err := c.saveOcclumConfig(hostPath); err != nil {
+		occlumConfigPath = filepath.Join(dataDir, c.configPath)
+		if err := c.saveOcclumConfig(occlumConfigPath); err != nil {
 			return "", err
 		}
 	}
-	cmd = append(cmd, "--occlum_config_path", occlumConfigPath)
-	logrus.Debugf("BuildUnsignedEnclave: command: %v", cmd)
+	logrus.Debugf("BuildUnsignedEnclave: command: %v", cmdArgs)
 	timeStart = time.Now()
-	if err := c.execTask(cmd...); err != nil {
-		logrus.Errorf("BuildUnsignedEnclave: exec failed. error: %++v", err)
+	cmdArgs = append(cmdArgs, "--occlum_config_path", occlumConfigPath)
+	if _, err := utils.ExecCommand("/bin/bash", cmdArgs...); err != nil {
+		logrus.Errorf("BuildUnsignedEnclave: execute command failed. error: %++v", err)
 		return "", err
 	}
 	logrus.Debugf("BuildUnsignedEnclave: init and build enclave time cost: %d", (time.Now().Sub(timeStart))/time.Second)
-	enclavePath := filepath.Join("/", rootfsDirName, c.workDirectory, "./build/lib/libocclum-libos.so")
+	enclavePath := filepath.Join(rootfsDir, c.workDirectory, "./build/lib/libocclum-libos.so")
 	logrus.Debugf("BuildUnsignedEnclave: total time cost: %d", (time.Now().Sub(ts))/time.Second)
+
 	return enclavePath, nil
 }
 
@@ -232,17 +133,21 @@ func (c *occlum) BuildUnsignedEnclave(req *task.CreateTaskRequest, args *carrier
 func (c *occlum) GenerateSigningMaterial(req *task.CreateTaskRequest, args *carrier.CommonArgs) (
 	signingMaterial string, err error) {
 	timeStart := time.Now()
-	signingMaterial = filepath.Join("/", rootfsDirName, c.workDirectory, "enclave_sig.dat")
-	args.Config = filepath.Join("/", rootfsDirName, c.workDirectory, "Enclave.xml")
-	cmd := []string{
-		"/bin/bash", filepath.Join("/", enclaveDataDir, carrierScriptFileName),
+	rootfsDir := filepath.Join(req.Bundle, rootfsDirName)
+	dataDir := filepath.Join(req.Bundle, dataDirName)
+	signingMaterial = filepath.Join(rootfsDir, c.workDirectory, "enclave_sig.dat")
+	args.Config = filepath.Join(rootfsDir, c.workDirectory, "Enclave.xml")
+	cmdArgs := []string{
+		filepath.Join(dataDir, carrierScriptFileName),
 		"--action", "generateSigningMaterial",
 		"--enclave_config_path", args.Config,
 		"--unsigned_encalve_path", args.Enclave,
 		"--unsigned_material_path", signingMaterial,
 	}
-	logrus.Debugf("GenerateSigningMaterial: sgx_sign gendata command: %v", cmd)
-	if err := c.execTask(cmd...); err != nil {
+	logrus.Debugf("GenerateSigningMaterial: sgx_sign gendata command: %v", cmdArgs)
+	//FIXME debug
+	time.Sleep(time.Minute * 2)
+	if _, err := utils.ExecCommand("/bin/bash", cmdArgs...); err != nil {
 		logrus.Errorf("GenerateSigningMaterial: sgx_sign gendata failed. error: %++v", err)
 		return "", err
 	}
@@ -255,30 +160,21 @@ func (c *occlum) GenerateSigningMaterial(req *task.CreateTaskRequest, args *carr
 func (c *occlum) CascadeEnclaveSignature(req *task.CreateTaskRequest, args *carrier.CascadeEnclaveSignatureArgs) (
 	signedEnclave string, err error) {
 	timeStart := time.Now()
-	var bufferSize int64 = 1024 * 4
-	signedEnclave = filepath.Join("/", rootfsDirName, c.workDirectory, "./build/lib/libocclum-libos.signed.so")
-	publicKey := filepath.Join("/", enclaveDataDir, "public_key.pem")
-	signature := filepath.Join("/", enclaveDataDir, "signature.dat")
-	if err := utils.CopyFile(args.Key, filepath.Join(req.Bundle, publicKey), bufferSize); err != nil {
-		logrus.Errorf("CascadeEnclaveSignature copy file %s to %s failed. err: %++v", args.Key, publicKey, err)
-		return "", err
-	}
-	if err := utils.CopyFile(args.Signature, filepath.Join(req.Bundle, signature), bufferSize); err != nil {
-		logrus.Errorf("CascadeEnclaveSignature copy file %s to %s failed. err: %++v", args.Signature, signature, err)
-		return "", err
-	}
-	cmd := []string{
-		"/bin/bash", filepath.Join("/", enclaveDataDir, carrierScriptFileName),
+	rootfsDir := filepath.Join(req.Bundle, rootfsDirName)
+	dataDir := filepath.Join(req.Bundle, dataDirName)
+	signedEnclave = filepath.Join(rootfsDir, c.workDirectory, "./build/lib/libocclum-libos.signed.so")
+	cmdArgs := []string{
+		filepath.Join(dataDir, carrierScriptFileName),
 		"--action", "cascadeEnclaveSignature",
 		"--enclave_config_path", args.Config,
 		"--unsigned_encalve_path", args.Enclave,
 		"--unsigned_material_path", args.SigningMaterial,
 		"--signed_enclave_path", signedEnclave,
-		"--public_key_path", publicKey,
-		"--signature_path", signature,
+		"--public_key_path", args.Key,
+		"--signature_path", args.Signature,
 	}
-	logrus.Debugf("CascadeEnclaveSignature: sgx_sign catsig command: %v", cmd)
-	if err := c.execTask(cmd...); err != nil {
+	logrus.Debugf("CascadeEnclaveSignature: sgx_sign catsig command: %v", cmdArgs)
+	if _, err := utils.ExecCommand("/bin/bash", cmdArgs...); err != nil {
 		logrus.Errorf("CascadeEnclaveSignature: sgx_sign catsig failed. error: %++v", err)
 		return "", err
 	}
@@ -289,7 +185,7 @@ func (c *occlum) CascadeEnclaveSignature(req *task.CreateTaskRequest, args *carr
 
 // Cleanup impl Carrier.
 func (c *occlum) Cleanup() error {
-	timeStart := time.Now()
+	/*timeStart := time.Now()
 	ts := timeStart
 	defer func() {
 		if c.task.client != nil {
@@ -344,7 +240,7 @@ func (c *occlum) Cleanup() error {
 		return err
 	}
 	logrus.Debugf("Cleanup: delete occlum SDK container task time cost: %d", (time.Now().Sub(timeStart))/time.Second)
-	logrus.Debugf("Cleanup: clean occlum container and task successfully")
+	logrus.Debugf("Cleanup: clean occlum container and task successfully")*/
 	return nil
 }
 

shim/runtime/utils/utils.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 )
 
 func CopyFile(src, dst string, bufferSize int64) error {
@@ -56,3 +57,12 @@ func GenerateID() string {
 	rand.Read(b)
 	return hex.EncodeToString(b)
 }
+
+// ExecCommand executes the cmd with args
+func ExecCommand(cmd string, arg ...string) ([]byte, error) {
+	b, err := exec.Command(cmd, arg...).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("%s %s", string(b), err)
+	}
+	return b, nil
+}

shim/runtime/v2/rune/v2/rune.go

@@ -54,7 +54,6 @@ func (s *service) carrierMain(req *taskAPI.CreateTaskRequest) (carrier.Carrier,
 		if carr, err = occlum.NewOcclumCarrier(s.context, req.Bundle); err != nil {
 			return nil, err
 		}
-
 		// mount rootfs
 		err = mountRootfs(req)
 		defer unmountRootfs(req)