rune: Code cleanup

Signed-off-by: Jia Zhang <zhang.jia@linux.alibaba.com>

rune/libcontainer/configs/enclave.go

@@ -10,5 +10,4 @@ type Enclave struct {
 	Type string `json:"type"`
 	Path string `json:"path"`
 	Args string `json:"args,omitempty"`
-	Signer string `json:"signer,omitempty"`
 }

rune/libcontainer/configs/validate/validator.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/intelrdt"
 	"github.com/opencontainers/runc/libenclave"
@@ -221,16 +220,7 @@ func (v *ConfigValidator) enclave(config *configs.Config) error {
 		return fmt.Errorf("enclave runtime path is not configured")
 	}
 
-	path := config.Enclave.Path
-	if config.Enclave.Signer != "server" {
-		var err error
-		path, err = securejoin.SecureJoin(config.Rootfs, path)
-		if err != nil {
-			return err
-		}
-	}
-
-	if _, err := os.Stat(path); err != nil {
+	if _, err := os.Stat(config.Enclave.Path); err != nil {
 		return err
 	}
 

rune/libcontainer/container_linux.go

@@ -528,23 +528,15 @@ func (c *linuxContainer) commandTemplate(p *Process, childInitPipe *os.File, chi
 		if agentPipe != nil {
 			cmd.ExtraFiles = append(cmd.ExtraFiles, agentPipe)
 			cmd.Env = append(cmd.Env,
-				fmt.Sprintf("_LIBCONTAINER_AGENTPIPE=%d", stdioFdCount+len(cmd.ExtraFiles)-1))
+				fmt.Sprintf("_LIBENCLAVE_AGENTPIPE=%d", stdioFdCount+len(cmd.ExtraFiles)-1))
 		}
 
 		if c.config.Enclave.Path != "" {
-			cmd.Env = append(cmd.Env, "_LIBCONTAINER_PAL_PATH="+c.config.Enclave.Path)
-		}
-
-		if c.config.Enclave.Signer != "server" {
-			cmd.Env = append(cmd.Env, "_LIBCONTAINER_PAL_ROOTFS="+c.config.Rootfs)
+			cmd.Env = append(cmd.Env, "_LIBENCLAVE_PAL_PATH="+c.config.Enclave.Path)
 		}
 
 		if detached {
-			cmd.Env = append(cmd.Env,
-				fmt.Sprintf("_LIBCONTAINER_DETACHED=%d", 1))
-		} else {
-			cmd.Env = append(cmd.Env,
-				fmt.Sprintf("_LIBCONTAINER_DETACHED=%d", 0))
+			cmd.Env = append(cmd.Env, "_LIBENCLAVE_DETACHED=1")
 		}
 	}
 

rune/libcontainer/factory_linux.go

@@ -346,8 +346,8 @@ func (l *LinuxFactory) StartInitialization() (err error) {
 		envConsole     = os.Getenv("_LIBCONTAINER_CONSOLE")
 		envLogPipe     = os.Getenv("_LIBCONTAINER_LOGPIPE")
 		envLogLevel    = os.Getenv("_LIBCONTAINER_LOGLEVEL")
-		envAgentPipe   = os.Getenv("_LIBCONTAINER_AGENTPIPE")
-		envDetached    = os.Getenv("_LIBCONTAINER_DETACHED")
+		envAgentPipe   = os.Getenv("_LIBENCLAVE_AGENTPIPE")
+		envDetached    = os.Getenv("_LIBENCLAVE_DETACHED")
 	)
 
 	// Get the INITPIPE.
@@ -391,7 +391,7 @@ func (l *LinuxFactory) StartInitialization() (err error) {
 	if envAgentPipe != "" {
 		agent, err := strconv.Atoi(envAgentPipe)
 		if err != nil {
-			return fmt.Errorf("unable to convert _LIBCONTAINER_AGENTSOCK=%s to int: %s", envAgentPipe, err)
+			return fmt.Errorf("unable to convert _LIBENCLAVE_AGENTSOCK=%s to int: %s", envAgentPipe, err)
 		}
 		agentPipe = os.NewFile(uintptr(agent), "agent-pipe")
 		defer agentPipe.Close()
@@ -400,7 +400,7 @@ func (l *LinuxFactory) StartInitialization() (err error) {
 	if envDetached != "" {
 		tmpDetached, err := strconv.Atoi(envDetached)
 		if err != nil {
-			return fmt.Errorf("unable to convert _LIBCONTAINER_DETACHED=%s to int: %s", envDetached, err)
+			return fmt.Errorf("unable to convert _LIBENCLAVE_DETACHED=%s to int: %s", envDetached, err)
 		}
 		if tmpDetached != 0 {
 			detached = true

rune/libcontainer/nsenter/loader.c → rune/libcontainer/nsenter/enclave.c

@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <dlfcn.h>
+#include <stdbool.h>
 #include <linux/limits.h>
 
 /* Defined in nsexec.c. */
@@ -15,7 +16,8 @@
 #define INFO    "info"
 #define DEBUG   "debug"
 
-void write_log_with_info(const char *level, const char *function, int line, const char *format, ...);
+void write_log_with_info(const char *level, const char *function,
+			 int line, const char *format, ...);
 
 #define write_log(level, fmt, ...) \
 	write_log_with_info((level), __FUNCTION__, __LINE__, (fmt), ##__VA_ARGS__)
@@ -27,58 +29,55 @@ void *fptr_pal_kill;
 void *fptr_pal_destroy;
 void *fptr_pal_create_process;
 
-int is_enclave(void)
+bool enclave_configured(void)
 {
-	const char *env;
-	env = getenv("_LIBCONTAINER_PAL_PATH");
-	if (env == NULL || *env == '\0')
-		return 0;
-	return 1;
+	const char *p = getenv("_LIBENCLAVE_PAL_PATH");
+	if (p == NULL || *p == '\0')
+		return false;
+	return true;
+}
+
+bool is_init_runelet(void)
+{
+	const char *type = getenv("_LIBCONTAINER_INITTYPE");
+        if (type == NULL || *type == '\0')
+                return false;
+	return !strcmp(type, "standard");
 }
 
 int load_enclave_runtime(void)
 {
-	char pal_full_path[PATH_MAX+1];
 	char *pal_path;
-	const char *rootfs;
 	void *dl;
 
-	pal_path = getenv("_LIBCONTAINER_PAL_PATH");
+	pal_path = getenv("_LIBENCLAVE_PAL_PATH");
 	if (pal_path == NULL || *pal_path == '\0') {
-		write_log(DEBUG, "invalid environment _LIBCONTAINER_PAL_PATH");
-		return 0;
+		write_log(ERROR, "_LIBENCLAVE_PAL_PATH should not be empty");
+		/* set errno correctly, make bail() work better */
+		errno = EINVAL;
+		return -1;
 	}
 
-	write_log(DEBUG, "_LIBCONTAINER_PAL_PATH = %s", pal_path);
-	write_log(DEBUG, "LD_LIBRARY_PATH = %s", getenv("LD_LIBRARY_PATH"));
-
-	rootfs = getenv("_LIBCONTAINER_PAL_ROOTFS");
-	if (rootfs && *rootfs != '\0') {
-		snprintf(pal_full_path, sizeof(pal_full_path) - 1, "%s/%s", rootfs, pal_path);
-		pal_path = pal_full_path;
-	}
+	write_log(DEBUG, "_LIBENCLAVE_PAL_PATH=%s", pal_path);
 
 	dl = dlopen(pal_path, RTLD_NOW);
-	unsetenv("LD_LIBRARY_PATH");
 	if (dl == NULL) {
-		write_log(DEBUG, "dlopen(): %s", dlerror());
-		/* set errno correctly, make bail() work better */
-		errno = ENOEXEC;
-		return -ENOEXEC;
+		write_log(ERROR, "failed to dlopen(): %s", dlerror());
+		return -1;
 	}
 
 #define DLSYM(fn)								\
 	do {									\
 		fptr_pal_ ## fn = dlsym(dl, "pal_" #fn);				\
-		write_log(DEBUG, "dlsym(%s) = %p", "pal_" #fn, fptr_pal_ ## fn);	\
+		write_log(DEBUG, "dlsym(%s)=%p", "pal_" #fn, fptr_pal_ ## fn);	\
 	} while (0)
 
 	DLSYM(get_version);
 	DLSYM(init);
+	DLSYM(create_process);
 	DLSYM(exec);
 	DLSYM(kill);
 	DLSYM(destroy);
-	DLSYM(create_process);
 #undef DLSYM
 
 	return 0;

rune/libcontainer/nsenter/nsexec.c

@@ -570,7 +570,8 @@ void join_namespaces(char *nslist)
 extern int ensure_cloned_binary(void);
 
 /* Defined in loader.c. */
-extern int is_enclave(void);
+extern bool enclave_configured(void);
+extern bool is_init_runelet(void);
 extern int load_enclave_runtime(void);
 
 void nsexec(void)
@@ -579,7 +580,6 @@ void nsexec(void)
 	jmp_buf env;
 	int sync_child_pipe[2], sync_grandchild_pipe[2];
 	struct nlconfig_t config = { 0 };
-	char *rootfs;
 
 	/*
 	 * Setup a pipe to send logs to the parent. This should happen
@@ -603,6 +603,12 @@ void nsexec(void)
 	if (ensure_cloned_binary() < 0)
 		bail("could not ensure we are a cloned binary");
 
+	if (enclave_configured()) {
+		int ret = load_enclave_runtime();
+		if (ret < 0)
+			bail("load_enclave_runtime() failed, ret = %d", ret);
+	}
+
 	write_log(DEBUG, "nsexec started");
 
 	/* Parse all of the netlink configuration. */
@@ -644,16 +650,6 @@ void nsexec(void)
 
 	/* TODO: Currently we aren't dealing with child deaths properly. */
 
-	rootfs = getenv("_LIBCONTAINER_PAL_ROOTFS");
-	if (rootfs && *rootfs != '\0') {
-		char ld_path[PATH_MAX+1];
-
-		snprintf(ld_path, sizeof(ld_path) - 1,
-			 "%s/usr/lib/x86_64-linux-gnu:%s/usr/lib:%s/usr/lib64:%s/lib:%s/lib64",
-			 rootfs, rootfs, rootfs, rootfs, rootfs);
-		setenv("LD_LIBRARY_PATH", ld_path, 1);
-	}
-
 	/*
 	 * Okay, so this is quite annoying.
 	 *
@@ -843,11 +839,6 @@ void nsexec(void)
 	case JUMP_CHILD:{
 			pid_t child;
 			enum sync_t s;
-			int ret;
-
-			ret = load_enclave_runtime();
-			if (ret < 0)
-				bail("load_enclave_runtime() failed, ret = %d", ret);
 
 			/* We're in a child and thus need to tell the parent if we die. */
 			syncfd = sync_child_pipe[0];
@@ -1040,8 +1031,17 @@ void nsexec(void)
 			/* Free netlink data. */
 			nl_free(&config);
 
-			if (is_enclave())
-				prctl(PR_SET_NAME, (unsigned long)"init-runelet", 0, 0, 0);
+			if (enclave_configured()) {
+				const char *name;
+
+				if (is_init_runelet())
+					name = "init-runelet";
+				else
+					name = "runelet";
+
+				/* For debugging. */
+				prctl(PR_SET_NAME, (unsigned long)name, 0, 0, 0);
+			}
 
 			/* Finish executing, let the Go runtime take over. */
 			return;

rune/libcontainer/process_linux.go

@@ -144,8 +144,6 @@ func (p *setnsProcess) start() (err error) {
 				Type: p.config.Config.Enclave.Type,
 				Path: p.config.Config.Enclave.Path,
 				Args: p.config.Config.Enclave.Args,
-				Signer: p.config.Config.Enclave.Signer,
-				Cmd:  p.process.Args,
 			}
 			err := utils.WriteJSON(p.messageSockPair.parent, config)
 			if err != nil {
@@ -476,8 +474,6 @@ func (p *initProcess) start() (retErr error) {
 				Type: p.config.Config.Enclave.Type,
 				Path: p.config.Config.Enclave.Path,
 				Args: p.config.Config.Enclave.Args,
-				Signer: p.config.Config.Enclave.Signer,
-				Cmd:  p.config.Args,
 			}
 			err := utils.WriteJSON(p.messageSockPair.parent, config)
 			if err != nil {

rune/libcontainer/setns_init_linux.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/sirupsen/logrus"
 	"github.com/opencontainers/runc/libcontainer/apparmor"
 	"github.com/opencontainers/runc/libcontainer/keys"
 	"github.com/opencontainers/runc/libcontainer/seccomp"
@@ -15,6 +14,7 @@ import (
 	"github.com/opencontainers/runc/libenclave"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 
 	"golang.org/x/sys/unix"
 )
@@ -95,20 +95,24 @@ func (l *linuxSetnsInit) Init() error {
 		}
 	}
 	if l.config.Config.Enclave != nil {
-		err := libenclave.StartBootstrap(l.pipe, l.logPipe, l.logLevel, -1, l.agentPipe, l.detached)
-		if err != nil {
-			return newSystemErrorWithCause(err, "libenclave bootstrap")
+		cfg := &libenclave.RuneletConfig{
+			InitPipe:  l.pipe,
+			LogPipe:   l.logPipe,
+			LogLevel:  l.logLevel,
+			FifoFd:    -1,
+			AgentPipe: l.agentPipe,
+			Detached:  l.detached,
 		}
 
-		exitCode, err := libenclave.StartInitialization()
+		exitCode, err := libenclave.StartInitialization(l.config.Args, cfg)
 		if err != nil {
 			logrus.Fatal(err)
 			os.Exit(1)
 		}
-		logrus.Debugf("enclave exitCode: %d", exitCode)
+		logrus.Debugf("enclave payload exit code: %d", exitCode)
 		os.Exit(int(exitCode))
 		// make compiler happy
-		return nil
+		return fmt.Errorf("failed to initialize runelet")
 	}
 	return system.Execv(l.config.Args[0], l.config.Args[0:], os.Environ())
 }

rune/libcontainer/specconv/spec_linux.go

@@ -331,17 +331,11 @@ func createEnclaveConfig(spec *specs.Spec, config *configs.Config) {
 		args = strings.Join(a, " ")
 	}
 
-	signer := filterOut(env, "ENCLAVE_RUNTIME_SIGNER")
-	if signer == "" {
-		signer = libcontainerUtils.SearchLabels(config.Labels, "enclave.runtime.signer")
-	}
-
 	if etype != "" {
 		config.Enclave = &configs.Enclave{
 			Type: etype,
 			Path: path,
 			Args: args,
-			Signer: signer,
 		}
 	}
 }

rune/libcontainer/standard_init_linux.go

@@ -9,7 +9,6 @@ import (
 	"runtime"
 	"syscall" //only for Exec
 
-	"github.com/sirupsen/logrus"
 	"github.com/opencontainers/runc/libcontainer/apparmor"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/keys"
@@ -18,6 +17,7 @@ import (
 	"github.com/opencontainers/runc/libenclave"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 
 	"golang.org/x/sys/unix"
 )
@@ -180,11 +180,30 @@ func (l *linuxStandardInit) Init() error {
 		return unix.Kill(unix.Getpid(), unix.SIGKILL)
 	}
 	if l.config.Config.Enclave != nil {
-		err := libenclave.StartBootstrap(l.pipe, l.logPipe, l.logLevel, l.fifoFd, l.agentPipe, l.detached)
+		if l.config.Config.Seccomp != nil && l.config.NoNewPrivileges {
+			if err := seccomp.InitSeccomp(l.config.Config.Seccomp); err != nil {
+				return newSystemErrorWithCause(err, "init seccomp")
+			}
+		}
+
+		cfg := &libenclave.RuneletConfig{
+			InitPipe:  l.pipe,
+			LogPipe:   l.logPipe,
+			LogLevel:  l.logLevel,
+			FifoFd:    l.fifoFd,
+			AgentPipe: l.agentPipe,
+			Detached:  l.detached,
+		}
+
+		exitCode, err := libenclave.StartInitialization(l.config.Args, cfg)
 		if err != nil {
-			return err
+			logrus.Fatal(err)
+			os.Exit(1)
 		}
-		return l.finalizeInit("/proc/self/exe", []string{"init-runelet", "enclave"}, true)
+		logrus.Debugf("init enclave runtime exit code: %d", exitCode)
+		os.Exit(int(exitCode))
+		// make compiler happy
+		return fmt.Errorf("failed to initialize init-runelet")
 	}
 	// Check for the arg before waiting to make sure it exists and it is
 	// returned as a create time error.
@@ -212,10 +231,6 @@ func (l *linuxStandardInit) Init() error {
 	// since been resolved.
 	// https://github.com/torvalds/linux/blob/v4.9/fs/exec.c#L1290-L1318
 	unix.Close(l.fifoFd)
-	return l.finalizeInit(name, l.config.Args[0:], false)
-}
-
-func (l *linuxStandardInit) finalizeInit(entryName string, args []string, noexec bool) error {
 	// Set seccomp as close to execve as possible, so as few syscalls take
 	// place afterward (reducing the amount of syscalls that users need to
 	// enable in their seccomp profiles).
@@ -224,18 +239,8 @@ func (l *linuxStandardInit) finalizeInit(entryName string, args []string, noexec
 			return newSystemErrorWithCause(err, "init seccomp")
 		}
 	}
-	if noexec {
-		exitCode, err := libenclave.StartInitialization()
-		if err != nil {
-			logrus.Fatal(err)
-			os.Exit(1)
-		}
-		logrus.Debugf("enclave exitCode: %d", exitCode)
-		os.Exit(int(exitCode))
-	} else {
-		if err := syscall.Exec(entryName, args, os.Environ()); err != nil {
-			return newSystemErrorWithCause(err, "exec user process")
-		}
+	if err := syscall.Exec(name, l.config.Args[0:], os.Environ()); err != nil {
+		return newSystemErrorWithCause(err, "exec user process")
 	}
-	return nil
+	return err
 }

rune/libenclave/bootstrap.go

-package libenclave // import "github.com/opencontainers/runc/libenclave"
-
-import (
-	"github.com/sirupsen/logrus"
-	"os"
-)
-
-type enclaveRuntimeEnv struct {
-	initPipe *os.File
-	logPipe *os.File
-	logLevel string
-	fifoFd int
-	agentPipe *os.File
-	detached bool
-}
-
-var enclaveEnv enclaveRuntimeEnv
-
-func GetEnclaveRunetimeEnv() *enclaveRuntimeEnv {
-	return &enclaveEnv
-}
-
-// `rune init` needs to execute self (/proc/self/exe) in container environment
-// as `runc init` executes entrypoint. Thus, some internal states in form of
-// environment variable must be staged and then recovered after re-exec. This
-// process is so called as libenclave bootstrapping, and the resulting process
-// is so called as runelet.
-func StartBootstrap(initPipe *os.File, logPipe *os.File, logLevel string, fifoFd int, agentPipe *os.File, detached bool) (err error) {
-	logrus.Debug("bootstrapping libenclave ...")
-
-	enclaveEnv.initPipe = initPipe
-	enclaveEnv.logPipe = logPipe
-	enclaveEnv.logLevel = logLevel
-	enclaveEnv.fifoFd = fifoFd
-	enclaveEnv.agentPipe = agentPipe
-	enclaveEnv.detached = detached
-
-	return nil
-}

rune/libenclave/configs/config.go

@@ -4,6 +4,4 @@ type InitEnclaveConfig struct {
 	Type string   `json:"type"`
 	Path string   `json:"path"`
 	Args string   `json:"args"`
-	Signer string `json:"signer"`
-	Cmd  []string `json:"cmd"`
 }

rune/libenclave/internal/runtime/enclave_runtime.go

@@ -7,6 +7,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"os"
 	"os/exec"
+	"strings"
 )
 
 type EnclaveRuntime interface {
@@ -62,7 +63,7 @@ func (rt *EnclaveRuntimeWrapper) LaunchAttestation() error {
 }
 
 func (rt *EnclaveRuntimeWrapper) ExecutePayload(cmd []string, envp []string, stdio [3]*os.File) (int32, error) {
-	logrus.Debugf("enclave runtime %s executing payload with commandline", cmd)
+	logrus.Debugf("enclave runtime %s executing payload with commandline", strings.Join(cmd, " "))
 
 	// The executable may not exist in container at all according
 	// to the design of enclave runtime, such as Occlum, which uses

rune/libenclave/runelet.go

@@ -21,17 +21,24 @@ const signalBufferSize = 2048
 
 var enclaveRuntime *runtime.EnclaveRuntimeWrapper
 
-func StartInitialization() (exitCode int32, err error) {
-	env := GetEnclaveRunetimeEnv()
+type RuneletConfig struct {
+	InitPipe  *os.File
+	LogPipe   *os.File
+	LogLevel  string
+	FifoFd    int
+	AgentPipe *os.File
+	Detached  bool
+}
 
-	logLevel := env.logLevel
+func StartInitialization(cmd []string, cfg *RuneletConfig) (exitCode int32, err error) {
+	logLevel := cfg.LogLevel
 
 	// Determine which type of runelet is initializing.
-	fifoFd := env.fifoFd
+	fifoFd := cfg.FifoFd
 
 	// Retrieve the init pipe fd to accomplish the enclave configuration
 	// handshake as soon as possible with parent rune.
-	initPipe := env.initPipe
+	initPipe := cfg.InitPipe
 	defer func() {
 		if err != nil {
 			initPipe.Close()
@@ -70,7 +77,7 @@ func StartInitialization() (exitCode int32, err error) {
 	}
 
 	// If runelet run as detach mode, close logrus before initpipe closed.
-	if env.detached {
+	if cfg.Detached {
 		logrus.SetOutput(ioutil.Discard)
 	}
 
@@ -84,13 +91,13 @@ func StartInitialization() (exitCode int32, err error) {
 	// and entrypoint, implying `rune exec` may preempt them too.
 
 	// Launch agent service for child runelet.
-	agentPipe := env.agentPipe
+	agentPipe := cfg.AgentPipe
 	defer agentPipe.Close()
 
 	notifySignal := make(chan os.Signal, signalBufferSize)
 
 	if fifoFd == -1 {
-		exitCode, err = remoteExec(agentPipe, config, notifySignal)
+		exitCode, err = remoteExec(agentPipe, cmd, notifySignal)
 		if err != nil {
 			return exitCode, err
 		}
@@ -116,7 +123,7 @@ func StartInitialization() (exitCode int32, err error) {
 	// have a way to prevent from this race happening.
 	enclaveRuntime = rt
 
-	exitCode, err = rt.ExecutePayload(config.Cmd, os.Environ(),
+	exitCode, err = rt.ExecutePayload(cmd, os.Environ(),
 		[3]*os.File{
 			os.Stdin, os.Stdout, os.Stderr,
 		})
@@ -191,12 +198,13 @@ func finalizeInitialization(fifoFd int) error {
 	return nil
 }
 
-func remoteExec(agentPipe *os.File, config *configs.InitEnclaveConfig, notifySignal chan os.Signal) (exitCode int32, err error) {
-	logrus.Debugf("preparing to remote exec %s", strings.Join(config.Cmd, " "))
+func remoteExec(agentPipe *os.File, cmd []string, notifySignal chan os.Signal) (exitCode int32, err error) {
+	c := strings.Join(cmd, " ")
+	logrus.Debugf("preparing to remote exec %s", c)
 
 	req := &pb.AgentServiceRequest{}
 	req.Exec = &pb.AgentServiceRequest_Execute{
-		Argv: strings.Join(config.Cmd, " "),
+		Argv: c,
 		Envp: strings.Join(os.Environ(), " "),
 	}
 	if err = protoBufWrite(agentPipe, req); err != nil {