diff --git a/shim/runtime/carrier/constants/constants.go b/shim/runtime/carrier/constants/constants.go index 62b11b83bbfc0900529512f2eb6ddcdb671615da..3c8c93ff02e75cb2988a56be96e1539bcedda36e 100644 --- a/shim/runtime/carrier/constants/constants.go +++ b/shim/runtime/carrier/constants/constants.go @@ -91,7 +91,7 @@ start $@` CarrierScript = `#!/bin/bash set -xe base_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" -occlum_workspace=/occlum_workspace +occlum_workspace=${base_dir}/occlum_workspace temp=$(getopt -a -o a:r:w:p:c:e:u:m:s:k:n: -l action:,rootfs:,work_dir:,entry_point:,occlum_config_path:,enclave_config_path:,\ unsigned_encalve_path:,unsigned_material_path:,signed_enclave_path:,public_key_path:,signature_path: -- "$@") @@ -151,7 +151,7 @@ function buildUnsignedEnclave(){ echo "BuildUnsignedEnclave:: the argumentes should not be empty: entry_point, rootfs, work_dir" exit 1 fi - + export PATH=$PATH:/opt/occlum/build/bin/ rm -fr ${occlum_workspace} mkdir -p ${occlum_workspace} pushd ${occlum_workspace} diff --git a/shim/runtime/carrier/occlum/occlum.go b/shim/runtime/carrier/occlum/occlum.go index 557796910b9f1c41c8a2d0db908eea4495657271..59005d0b2290fa0cd737fa4e35f0b173e952a78e 100644 --- a/shim/runtime/carrier/occlum/occlum.go +++ b/shim/runtime/carrier/occlum/occlum.go @@ -5,11 +5,8 @@ import ( "encoding/json" "fmt" "io/ioutil" - "math/rand" "os" "path/filepath" - "strconv" - "syscall" "time" "github.com/BurntSushi/toml" @@ -22,8 +19,6 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/cio" "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/oci" "github.com/containerd/containerd/runtime/v2/task" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -31,11 +26,11 @@ import ( const ( defaultNamespace = "k8s.io" + startScriptFileName = "start.sh" replaceOcclumImageScript = "replace_occlum_image.sh" carrierScriptFileName = "carrier.sh" - startScriptFileName = "start.sh" rootfsDirName = "rootfs" - enclaveDataDir = "data" + dataDirName = "data" ) var _ carrier.Carrier = &occlum{} @@ -87,144 +82,50 @@ func (c *occlum) BuildUnsignedEnclave(req *task.CreateTaskRequest, args *carrier return "", err } - namespace, ok := namespaces.Namespace(c.context) - if !ok { - namespace = defaultNamespace - } - // Create a new client connected to the default socket path for containerd. - client, err := containerd.New(c.shimConfig.Containerd.Socket) - if err != nil { - return "", fmt.Errorf("failed to create containerd client. error: %++v", err) - } else { - c.task.client = client - } - logrus.Debugf("BuildUnsignedEnclave: create containerd client time cost: %d", (time.Now().Sub(timeStart))/time.Second) - logrus.Debugf("BuildUnsignedEnclave: get containerd client successfully") - - if err = createNamespaceIfNotExist(client, namespace); err != nil { - logrus.Errorf("BuildUnsignedEnclave: create namespace %s failed. error: %++v", namespace, err) - return "", err - } - - // pull the image that used to build enclave. - occlumEnclaveBuilderImage := c.shimConfig.EnclaveRuntime.Occlum.BuildImage - timeStart = time.Now() - image, err := client.Pull(c.context, occlumEnclaveBuilderImage, containerd.WithPullUnpack) - if err != nil { - return "", fmt.Errorf("failed to pull image %s. error: %++v", occlumEnclaveBuilderImage, err) - } - logrus.Debugf("BuildUnsignedEnclave: pull occlum SDK image time cost: %d", (time.Now().Sub(timeStart))/time.Second) - logrus.Debugf("BuildUnsignedEnclave: pull image %s successfully", occlumEnclaveBuilderImage) - - // Generate the containerId and snapshotId. - // FIXME The variables containerId and snapshotId should be generated by utils.GenerateID - rand.Seed(time.Now().UnixNano()) - containerId := fmt.Sprintf("occlum-enclave-builder-%s", strconv.FormatInt(rand.Int63(), 16)) - snapshotId := fmt.Sprintf("occlum-enclave-builder-snapshot-%s", strconv.FormatInt(rand.Int63(), 16)) + // Copy the script files that are used to build encalve.so by occlum into rootfs + rootfsDir := filepath.Join(req.Bundle, rootfsDirName) + dataDir := filepath.Join(req.Bundle, dataDirName) + os.MkdirAll(dataDir, 0755) - logrus.Debugf("BuildUnsignedEnclave: containerId: %s, snapshotId: %s", containerId, snapshotId) - - if err := os.Mkdir(filepath.Join(req.Bundle, enclaveDataDir), 0755); err != nil { - return "", err - } - - replaceImagesScript := filepath.Join(req.Bundle, enclaveDataDir, replaceOcclumImageScript) + replaceImagesScript := filepath.Join(dataDir, replaceOcclumImageScript) if err := ioutil.WriteFile(replaceImagesScript, []byte(carr_const.ReplaceOcclumImageScript), os.ModePerm); err != nil { return "", err } - carrierScript := filepath.Join(req.Bundle, enclaveDataDir, carrierScriptFileName) + carrierScript := filepath.Join(dataDir, carrierScriptFileName) if err := ioutil.WriteFile(carrierScript, []byte(carr_const.CarrierScript), os.ModePerm); err != nil { return "", err } - startScript := filepath.Join(req.Bundle, enclaveDataDir, startScriptFileName) - if err := ioutil.WriteFile(startScript, []byte(carr_const.StartScript), os.ModePerm); err != nil { - return "", err - } - - // Create rootfs mount points. - mounts := make([]specs.Mount, 0) - rootfsMount := specs.Mount{ - Destination: filepath.Join("/", rootfsDirName), - Type: "bind", - Source: filepath.Join(req.Bundle, rootfsDirName), - Options: []string{"rbind", "rw"}, - } - dataMount := specs.Mount{ - Destination: filepath.Join("/", enclaveDataDir), - Type: "bind", - Source: filepath.Join(req.Bundle, enclaveDataDir), - Options: []string{"rbind", "rw"}, - } - - logrus.Debugf("BuildUnsignedEnclave: rootfsMount source: %s, destination: %s", - rootfsMount.Source, rootfsMount.Destination) - - mounts = append(mounts, rootfsMount, dataMount) - // create a container - timeStart = time.Now() - container, err := client.NewContainer( - c.context, - containerId, - containerd.WithImage(image), - containerd.WithNewSnapshot(snapshotId, image), - containerd.WithNewSpec(oci.WithImageConfig(image), - oci.WithProcessArgs("/bin/bash", filepath.Join("/", enclaveDataDir, startScriptFileName)), - oci.WithPrivileged, - oci.WithMounts(mounts), - ), - ) - if err != nil { - return "", fmt.Errorf("failed to create container by image %s. error: %++v", - occlumEnclaveBuilderImage, err) - } else { - c.task.container = &container - } - logrus.Debugf("BuildUnsignedEnclave: create occlum SDK container time cost: %d", (time.Now().Sub(timeStart))/time.Second) - - // Create a task from the container. - t, err := container.NewTask(c.context, cio.NewCreator(cio.WithStdio)) - if err != nil { - return "", err - } else { - c.task.task = &t - } - logrus.Debugf("BuildUnsignedEnclave: create task successfully") - - if err := t.Start(c.context); err != nil { - logrus.Errorf("BuildUnsignedEnclave: start task failed. error: %++v", err) - return "", err - } - - cmd := []string{ - "/bin/bash", filepath.Join("/", enclaveDataDir, carrierScriptFileName), + // Execute the carrier script to generate the unsigned enclave.so in rootfs + cmdArgs := []string{ + "/bin/bash", filepath.Join(dataDir, carrierScriptFileName), "--action", "buildUnsignedEnclave", "--entry_point", c.entryPoints[0], "--work_dir", c.workDirectory, - "--rootfs", filepath.Join("/", rootfsDirName), + "--rootfs", rootfsDir, } var occlumConfigPath string if c.configPath != "" { - occlumConfigPath = filepath.Join("/", rootfsDirName, c.configPath) + occlumConfigPath = filepath.Join(rootfsDir, c.configPath) } else { c.configPath = "Occlum.json" - occlumConfigPath = filepath.Join("/", enclaveDataDir, c.configPath) - hostPath := filepath.Join(c.bundle, enclaveDataDir, c.configPath) - if err := c.saveOcclumConfig(hostPath); err != nil { + occlumConfigPath = filepath.Join(dataDir, c.configPath) + if err := c.saveOcclumConfig(occlumConfigPath); err != nil { return "", err } } - cmd = append(cmd, "--occlum_config_path", occlumConfigPath) - logrus.Debugf("BuildUnsignedEnclave: command: %v", cmd) + logrus.Debugf("BuildUnsignedEnclave: command: %v", cmdArgs) timeStart = time.Now() - if err := c.execTask(cmd...); err != nil { - logrus.Errorf("BuildUnsignedEnclave: exec failed. error: %++v", err) + cmdArgs = append(cmdArgs, "--occlum_config_path", occlumConfigPath) + if _, err := utils.ExecCommand("/bin/bash", cmdArgs...); err != nil { + logrus.Errorf("BuildUnsignedEnclave: execute command failed. error: %++v", err) return "", err } logrus.Debugf("BuildUnsignedEnclave: init and build enclave time cost: %d", (time.Now().Sub(timeStart))/time.Second) - enclavePath := filepath.Join("/", rootfsDirName, c.workDirectory, "./build/lib/libocclum-libos.so") + enclavePath := filepath.Join(rootfsDir, c.workDirectory, "./build/lib/libocclum-libos.so") logrus.Debugf("BuildUnsignedEnclave: total time cost: %d", (time.Now().Sub(ts))/time.Second) + return enclavePath, nil } @@ -232,17 +133,21 @@ func (c *occlum) BuildUnsignedEnclave(req *task.CreateTaskRequest, args *carrier func (c *occlum) GenerateSigningMaterial(req *task.CreateTaskRequest, args *carrier.CommonArgs) ( signingMaterial string, err error) { timeStart := time.Now() - signingMaterial = filepath.Join("/", rootfsDirName, c.workDirectory, "enclave_sig.dat") - args.Config = filepath.Join("/", rootfsDirName, c.workDirectory, "Enclave.xml") - cmd := []string{ - "/bin/bash", filepath.Join("/", enclaveDataDir, carrierScriptFileName), + rootfsDir := filepath.Join(req.Bundle, rootfsDirName) + dataDir := filepath.Join(req.Bundle, dataDirName) + signingMaterial = filepath.Join(rootfsDir, c.workDirectory, "enclave_sig.dat") + args.Config = filepath.Join(rootfsDir, c.workDirectory, "Enclave.xml") + cmdArgs := []string{ + filepath.Join(dataDir, carrierScriptFileName), "--action", "generateSigningMaterial", "--enclave_config_path", args.Config, "--unsigned_encalve_path", args.Enclave, "--unsigned_material_path", signingMaterial, } - logrus.Debugf("GenerateSigningMaterial: sgx_sign gendata command: %v", cmd) - if err := c.execTask(cmd...); err != nil { + logrus.Debugf("GenerateSigningMaterial: sgx_sign gendata command: %v", cmdArgs) + //FIXME debug + time.Sleep(time.Minute * 2) + if _, err := utils.ExecCommand("/bin/bash", cmdArgs...); err != nil { logrus.Errorf("GenerateSigningMaterial: sgx_sign gendata failed. error: %++v", err) return "", err } @@ -255,30 +160,21 @@ func (c *occlum) GenerateSigningMaterial(req *task.CreateTaskRequest, args *carr func (c *occlum) CascadeEnclaveSignature(req *task.CreateTaskRequest, args *carrier.CascadeEnclaveSignatureArgs) ( signedEnclave string, err error) { timeStart := time.Now() - var bufferSize int64 = 1024 * 4 - signedEnclave = filepath.Join("/", rootfsDirName, c.workDirectory, "./build/lib/libocclum-libos.signed.so") - publicKey := filepath.Join("/", enclaveDataDir, "public_key.pem") - signature := filepath.Join("/", enclaveDataDir, "signature.dat") - if err := utils.CopyFile(args.Key, filepath.Join(req.Bundle, publicKey), bufferSize); err != nil { - logrus.Errorf("CascadeEnclaveSignature copy file %s to %s failed. err: %++v", args.Key, publicKey, err) - return "", err - } - if err := utils.CopyFile(args.Signature, filepath.Join(req.Bundle, signature), bufferSize); err != nil { - logrus.Errorf("CascadeEnclaveSignature copy file %s to %s failed. err: %++v", args.Signature, signature, err) - return "", err - } - cmd := []string{ - "/bin/bash", filepath.Join("/", enclaveDataDir, carrierScriptFileName), + rootfsDir := filepath.Join(req.Bundle, rootfsDirName) + dataDir := filepath.Join(req.Bundle, dataDirName) + signedEnclave = filepath.Join(rootfsDir, c.workDirectory, "./build/lib/libocclum-libos.signed.so") + cmdArgs := []string{ + filepath.Join(dataDir, carrierScriptFileName), "--action", "cascadeEnclaveSignature", "--enclave_config_path", args.Config, "--unsigned_encalve_path", args.Enclave, "--unsigned_material_path", args.SigningMaterial, "--signed_enclave_path", signedEnclave, - "--public_key_path", publicKey, - "--signature_path", signature, + "--public_key_path", args.Key, + "--signature_path", args.Signature, } - logrus.Debugf("CascadeEnclaveSignature: sgx_sign catsig command: %v", cmd) - if err := c.execTask(cmd...); err != nil { + logrus.Debugf("CascadeEnclaveSignature: sgx_sign catsig command: %v", cmdArgs) + if _, err := utils.ExecCommand("/bin/bash", cmdArgs...); err != nil { logrus.Errorf("CascadeEnclaveSignature: sgx_sign catsig failed. error: %++v", err) return "", err } @@ -289,7 +185,7 @@ func (c *occlum) CascadeEnclaveSignature(req *task.CreateTaskRequest, args *carr // Cleanup impl Carrier. func (c *occlum) Cleanup() error { - timeStart := time.Now() + /*timeStart := time.Now() ts := timeStart defer func() { if c.task.client != nil { @@ -344,7 +240,7 @@ func (c *occlum) Cleanup() error { return err } logrus.Debugf("Cleanup: delete occlum SDK container task time cost: %d", (time.Now().Sub(timeStart))/time.Second) - logrus.Debugf("Cleanup: clean occlum container and task successfully") + logrus.Debugf("Cleanup: clean occlum container and task successfully")*/ return nil } diff --git a/shim/runtime/utils/utils.go b/shim/runtime/utils/utils.go index 0895284e55777a4df371adf54991c4d6ebf2b947..bec8b9e0fdfb0f26f3762120836747e4ccaa55a8 100644 --- a/shim/runtime/utils/utils.go +++ b/shim/runtime/utils/utils.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "os/exec" ) func CopyFile(src, dst string, bufferSize int64) error { @@ -56,3 +57,12 @@ func GenerateID() string { rand.Read(b) return hex.EncodeToString(b) } + +// ExecCommand executes the cmd with args +func ExecCommand(cmd string, arg ...string) ([]byte, error) { + b, err := exec.Command(cmd, arg...).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("%s %s", string(b), err) + } + return b, nil +} diff --git a/shim/runtime/v2/rune/v2/rune.go b/shim/runtime/v2/rune/v2/rune.go index 75b124b54731f5898be1d7fac2d2410fab36532a..2caaddc0c861f7ba65c2410f1519f08cf971ed45 100644 --- a/shim/runtime/v2/rune/v2/rune.go +++ b/shim/runtime/v2/rune/v2/rune.go @@ -54,7 +54,6 @@ func (s *service) carrierMain(req *taskAPI.CreateTaskRequest) (carrier.Carrier, if carr, err = occlum.NewOcclumCarrier(s.context, req.Bundle); err != nil { return nil, err } - // mount rootfs err = mountRootfs(req) defer unmountRootfs(req)