未验证 提交 d2e29fc8 编写于 作者: T tianjia 提交者: GitHub

rune: Support to load enclave runtime at bootstrap

Instead loading enclave runtime in container, rune should always load it at bootstrap,
in order to avoid dlopen issue.
Signed-off-by: NTianjia Zhang <tianjia.zhang@linux.alibaba.com>
上级 174504be
package main
import (
"fmt"
"os"
"runtime"
"github.com/opencontainers/runc/libcontainer/logs"
"github.com/opencontainers/runc/libenclave"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
func init() {
if len(os.Args) > 1 && os.Args[1] == "enclave" {
runtime.GOMAXPROCS(1)
runtime.LockOSThread()
level := os.Getenv("_LIBENCLAVE_LOGLEVEL")
logLevel, err := logs.ParseLogLevel(level)
if err != nil {
panic(fmt.Sprintf("runelet: failed to parse log level: %q: %v", level,
err))
}
err = logs.ConfigureLogging(logs.Config{
LogPipeFd: os.Getenv("_LIBENCLAVE_LOGPIPE"),
LogFormat: "json",
LogLevel: logLevel,
})
if err != nil {
panic(fmt.Sprintf("runelet: failed to configure logging: %v", err))
}
logrus.Debug("runelet process started")
}
}
var enclaveCommand = cli.Command{
Name: "enclave",
Usage: `initialize the enclave runtime (do not call it outside of rune)`,
Action: func(context *cli.Context) error {
exitCode, err := libenclave.StartInitialization()
if err != nil {
logrus.Fatal(err)
}
os.Exit(int(exitCode))
panic("runelet process failed to exit")
},
}
......@@ -10,4 +10,5 @@ type Enclave struct {
Type string `json:"type"`
Path string `json:"path"`
Args string `json:"args,omitempty"`
Signer string `json:"signer,omitempty"`
}
......@@ -221,10 +221,14 @@ func (v *ConfigValidator) enclave(config *configs.Config) error {
return fmt.Errorf("enclave runtime path is not configured")
}
path, err := securejoin.SecureJoin(config.Rootfs, config.Enclave.Path)
path := config.Enclave.Path
if config.Enclave.Signer != "server" {
var err error
path, err = securejoin.SecureJoin(config.Rootfs, config.Enclave.Path)
if err != nil {
return err
}
}
if _, err := os.Stat(path); err != nil {
return err
......
......@@ -530,6 +530,13 @@ func (c *linuxContainer) commandTemplate(p *Process, childInitPipe *os.File, chi
cmd.Env = append(cmd.Env,
fmt.Sprintf("_LIBCONTAINER_AGENTPIPE=%d", stdioFdCount+len(cmd.ExtraFiles)-1))
}
if c.config.Enclave.Path != "" {
cmd.Env = append(cmd.Env, "_LIBCONTAINER_PAL_PATH=" + string(c.config.Enclave.Path))
}
if c.config.Enclave.Signer != "server" {
cmd.Env = append(cmd.Env, "_LIBCONTAINER_PAL_ROOTFS=" + string(c.config.Rootfs))
}
}
// NOTE: when running a container with no PID namespace and the parent process spawning the container is
......
#define _GNU_SOURCE
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dlfcn.h>
/* Defined in nsexec.c. */
#define PANIC "panic"
#define FATAL "fatal"
#define ERROR "error"
#define WARNING "warning"
#define INFO "info"
#define DEBUG "debug"
void write_log_with_info(const char *level, const char *function, int line, const char *format, ...);
#define write_log(level, fmt, ...) \
write_log_with_info((level), __FUNCTION__, __LINE__, (fmt), ##__VA_ARGS__)
struct pal_attr_t {
const char *args;
const char *log_level;
};
struct pal_stdio_fds {
int stdin, stdout, stderr;
};
int *pal_version;
int (*fptr_pal_init)(const struct pal_attr_t *attr);
int (*fptr_pal_exec)(const char *path, const char * const argv[],
const struct pal_stdio_fds *stdio, int *exit_code);
int (*fptr_pal_kill)(int sig, int pid);
int (*fptr_pal_destroy)(void);
#define PAL_SO_PREFIX "liberpal-"
#define PAL_SO_SUFFIX ".so"
int is_enclave(void)
{
const char *env;
env = getenv("_LIBCONTAINER_PAL_PATH");
if (env == NULL || *env == '\0')
return 0;
return 1;
}
int load_enclave_runtime(void)
{
const char *file, *basename, *suffix, *name;
int namelen;
const char *rootfs;
void *dl;
file = getenv("_LIBCONTAINER_PAL_PATH");
if (file == NULL || *file == '\0') {
write_log(DEBUG, "invalid environment _LIBCONTAINER_PAL_PATH");
return -EINVAL;
}
write_log(DEBUG, "_LIBCONTAINER_PAL_PATH = %s", file);
/* fetch basename */
basename = strrchr(file, '/');
if (basename)
basename += 1; /* skip '/' */
else
basename = file;
/* check prefix and suffix */
if (strncmp(basename, PAL_SO_PREFIX, sizeof(PAL_SO_PREFIX) - 1) != 0)
return -ESRCH;
suffix = basename + strlen(basename) - sizeof(PAL_SO_SUFFIX) + 1;
if (strncmp(suffix, PAL_SO_SUFFIX, sizeof(PAL_SO_SUFFIX) - 1) != 0)
return -ESRCH;
/* pal name */
name = basename + sizeof(PAL_SO_PREFIX) - 1;
namelen = strlen(name) - sizeof(PAL_SO_SUFFIX) + 1;
/* dlopen */
rootfs = getenv("_LIBCONTAINER_PAL_ROOTFS");
if (rootfs && *rootfs != '\0') {
char sofile[BUFSIZ];
char ldpath[BUFSIZ];
const char *env_ldpath;
if (basename == file) {
write_log(DEBUG, "_LIBCONTAINER_PAL_PATH must be a absolute path");
return -ENOSPC;
}
snprintf(sofile, sizeof(sofile), "%s/%s", rootfs, file);
snprintf(ldpath, sizeof(ldpath), "%s/lib64", rootfs);
env_ldpath = getenv("LD_LIBRARY_PATH");
if (env_ldpath && *env_ldpath != '\0') {
char *saved_ldpath = strdup(env_ldpath);
if (saved_ldpath == NULL)
return -ENOMEM;
setenv("LD_LIBRARY_PATH", ldpath, 1);
dl = dlopen(sofile, RTLD_NOW);
setenv("LD_LIBRARY_PATH", saved_ldpath, 1);
free(saved_ldpath);
} else {
setenv("LD_LIBRARY_PATH", ldpath, 1);
dl = dlopen(sofile, RTLD_NOW);
unsetenv("LD_LIBRARY_PATH");
}
} else {
dl = dlopen(file, RTLD_NOW);
}
if (dl == NULL) {
write_log(DEBUG, "dlopen(): %s", dlerror());
return -ENOEXEC;
}
pal_version = dlsym(dl, "pal_version");
write_log(DEBUG, "dlsym(%s) = %p", "pal_version", pal_version);
#define DLSYM(fn) \
do { \
char fname[64]; \
snprintf(fname, sizeof(fname), "%.*s_pal_%s", namelen, name, #fn); \
fptr_pal_ ## fn = dlsym(dl, fname); \
write_log(DEBUG, "dlsym(%s) = %p", fname, fptr_pal_ ## fn); \
} while (0)
DLSYM(init);
DLSYM(exec);
DLSYM(kill);
DLSYM(destroy);
#undef DLSYM
return 0;
}
// +build linux
package nsenter
/*
#cgo LDFLAGS: -ldl
#include <stdio.h>
#include <stdlib.h>
struct pal_attr_t {
const char *args;
const char *log_level;
};
struct pal_stdio_fds {
int stdin, stdout, stderr;
};
extern int *pal_version;
extern int (*fptr_pal_init)(const struct pal_attr_t *attr);
extern int (*fptr_pal_exec)(const char *path, const char * const argv[],
const struct pal_stdio_fds *stdio, int *exit_code);
extern int (*fptr_pal_kill)(int sig, int pid);
extern int (*fptr_pal_destroy)(void);
*/
import "C"
import (
"unsafe"
)
func SymAddrPalVersion() unsafe.Pointer {
return unsafe.Pointer(C.pal_version)
}
func SymAddrPalInit() unsafe.Pointer {
return unsafe.Pointer(C.fptr_pal_init)
}
func SymAddrPalExec() unsafe.Pointer {
return unsafe.Pointer(C.fptr_pal_exec)
}
func SymAddrPalKill() unsafe.Pointer {
return unsafe.Pointer(C.fptr_pal_kill)
}
func SymAddrPalDestroy() unsafe.Pointer {
return unsafe.Pointer(C.fptr_pal_destroy)
}
......@@ -134,7 +134,7 @@ int setns(int fd, int nstype)
}
#endif
static void write_log_with_info(const char *level, const char *function, int line, const char *format, ...)
void write_log_with_info(const char *level, const char *function, int line, const char *format, ...)
{
char message[1024] = {};
......@@ -569,6 +569,10 @@ void join_namespaces(char *nslist)
/* Defined in cloned_binary.c. */
extern int ensure_cloned_binary(void);
/* Defined in loader.c. */
extern int is_enclave(void);
extern int load_enclave_runtime(void);
void nsexec(void)
{
int pipenum;
......@@ -698,10 +702,15 @@ void nsexec(void)
int len;
pid_t child, first_child = -1;
bool ready = false;
int ret;
/* For debugging. */
prctl(PR_SET_NAME, (unsigned long)"runc:[0:PARENT]", 0, 0, 0);
ret = load_enclave_runtime();
if (ret < 0)
bail("load_enclave_runtime() failed, ret = %d", ret);
/* Start the process of getting a container. */
child = clone_parent(&env, JUMP_CHILD);
if (child < 0)
......@@ -1020,6 +1029,9 @@ void nsexec(void)
/* Free netlink data. */
nl_free(&config);
if (is_enclave())
prctl(PR_SET_NAME, (unsigned long)"init-runelet", 0, 0, 0);
/* Finish executing, let the Go runtime take over. */
return;
}
......
......@@ -144,6 +144,7 @@ func (p *setnsProcess) start() (err error) {
Type: p.config.Config.Enclave.Type,
Path: p.config.Config.Enclave.Path,
Args: p.config.Config.Enclave.Args,
Signer: p.config.Config.Enclave.Signer,
Cmd: p.process.Args,
}
err := utils.WriteJSON(p.messageSockPair.parent, config)
......@@ -475,6 +476,7 @@ func (p *initProcess) start() (retErr error) {
Type: p.config.Config.Enclave.Type,
Path: p.config.Config.Enclave.Path,
Args: p.config.Config.Enclave.Args,
Signer: p.config.Config.Enclave.Signer,
Cmd: p.config.Args,
}
err := utils.WriteJSON(p.messageSockPair.parent, config)
......
......@@ -7,6 +7,7 @@ import (
"os"
"runtime"
"github.com/sirupsen/logrus"
"github.com/opencontainers/runc/libcontainer/apparmor"
"github.com/opencontainers/runc/libcontainer/keys"
"github.com/opencontainers/runc/libcontainer/seccomp"
......@@ -97,7 +98,16 @@ func (l *linuxSetnsInit) Init() error {
if err != nil {
return newSystemErrorWithCause(err, "libenclave bootstrap")
}
return system.Execv("/proc/self/exe", []string{"runelet", "enclave"}, os.Environ())
exitCode, err := libenclave.StartInitialization()
if err != nil {
logrus.Fatal(err)
os.Exit(1)
}
logrus.Debugf("enclave exitCode: %d", exitCode)
os.Exit(int(exitCode))
// make compiler happy
return nil
}
return system.Execv(l.config.Args[0], l.config.Args[0:], os.Environ())
}
......@@ -331,11 +331,17 @@ func createEnclaveConfig(spec *specs.Spec, config *configs.Config) {
args = strings.Join(a, " ")
}
signer := filterOut(env, "ENCLAVE_RUNTIME_SIGNER")
if signer == "" {
signer = libcontainerUtils.SearchLabels(config.Labels, "enclave.runtime.signer")
}
if etype != "" {
config.Enclave = &configs.Enclave{
Type: etype,
Path: path,
Args: args,
Signer: signer,
}
}
}
......
......@@ -9,6 +9,7 @@ import (
"runtime"
"syscall" //only for Exec
"github.com/sirupsen/logrus"
"github.com/opencontainers/runc/libcontainer/apparmor"
"github.com/opencontainers/runc/libcontainer/configs"
"github.com/opencontainers/runc/libcontainer/keys"
......@@ -182,7 +183,7 @@ func (l *linuxStandardInit) Init() error {
if err != nil {
return err
}
return l.finalizeInit("/proc/self/exe", []string{"init-runelet", "enclave"})
return l.finalizeInit("/proc/self/exe", []string{"init-runelet", "enclave"}, true)
}
// Check for the arg before waiting to make sure it exists and it is
// returned as a create time error.
......@@ -210,10 +211,10 @@ func (l *linuxStandardInit) Init() error {
// since been resolved.
// https://github.com/torvalds/linux/blob/v4.9/fs/exec.c#L1290-L1318
unix.Close(l.fifoFd)
return l.finalizeInit(name, l.config.Args[0:])
return l.finalizeInit(name, l.config.Args[0:], false)
}
func (l *linuxStandardInit) finalizeInit(entryName string, args []string) error {
func (l *linuxStandardInit) finalizeInit(entryName string, args []string, noexec bool) error {
// Set seccomp as close to execve as possible, so as few syscalls take
// place afterward (reducing the amount of syscalls that users need to
// enable in their seccomp profiles).
......@@ -222,8 +223,18 @@ func (l *linuxStandardInit) finalizeInit(entryName string, args []string) error
return newSystemErrorWithCause(err, "init seccomp")
}
}
if noexec {
exitCode, err := libenclave.StartInitialization()
if err != nil {
logrus.Fatal(err)
os.Exit(1)
}
logrus.Debugf("enclave exitCode: %d", exitCode)
os.Exit(int(exitCode))
} else {
if err := syscall.Exec(entryName, args, os.Environ()); err != nil {
return newSystemErrorWithCause(err, "exec user process")
}
}
return nil
}
......@@ -4,5 +4,6 @@ type InitEnclaveConfig struct {
Type string `json:"type"`
Path string `json:"path"`
Args string `json:"args"`
Signer string `json:"signer"`
Cmd []string `json:"cmd"`
}
......@@ -50,6 +50,8 @@ import (
"os"
"strings"
"unsafe"
"github.com/opencontainers/runc/libcontainer/nsenter"
)
const (
......@@ -59,7 +61,17 @@ const (
type enclaveRuntimePalApiV1 struct {
}
func (api *enclaveRuntimePalApiV1) init(sym unsafe.Pointer, args string, logLevel string) error {
func (pal *enclaveRuntimePalApiV1) get_version() uint32 {
logrus.Debugf("pal get_version() called")
sym := nsenter.SymAddrPalVersion()
if sym != nil {
return *(*uint32)(sym)
} else {
return palApiVersion
}
}
func (api *enclaveRuntimePalApiV1) init(args string, logLevel string) error {
logrus.Debugf("pal init() called with args %s", args)
a := C.CString(args)
......@@ -68,6 +80,7 @@ func (api *enclaveRuntimePalApiV1) init(sym unsafe.Pointer, args string, logLeve
l := C.CString(logLevel)
defer C.free(unsafe.Pointer(l))
sym := nsenter.SymAddrPalInit()
ret := C.palInitV1(sym, a, l)
if ret < 0 {
return fmt.Errorf("pal init() failed with %d", ret)
......@@ -75,7 +88,7 @@ func (api *enclaveRuntimePalApiV1) init(sym unsafe.Pointer, args string, logLeve
return nil
}
func (pal *enclaveRuntimePalApiV1) exec(sym unsafe.Pointer, cmd []string, envs []string, stdio [3]*os.File) (int32, error) {
func (pal *enclaveRuntimePalApiV1) exec(cmd []string, envs []string, stdio [3]*os.File) (int32, error) {
logrus.Debugf("pal exec() called with args %s", strings.Join(cmd, " "))
// Skip cmd[0] as used as the executable.
......@@ -109,6 +122,7 @@ func (pal *enclaveRuntimePalApiV1) exec(sym unsafe.Pointer, cmd []string, envs [
stdin := C.int(int(stdio[0].Fd()))
stdout := C.int(int(stdio[1].Fd()))
stderr := C.int(int(stdio[2].Fd()))
sym := nsenter.SymAddrPalExec()
ret := C.palExecV1(sym, exe, argv, envp, (*C.int)(unsafe.Pointer(&exitCode)), stdin, stdout, stderr)
if ret < 0 {
return exitCode, fmt.Errorf("pal exec() failed with %d", ret)
......@@ -116,9 +130,10 @@ func (pal *enclaveRuntimePalApiV1) exec(sym unsafe.Pointer, cmd []string, envs [
return exitCode, nil
}
func (pal *enclaveRuntimePalApiV1) kill(sym unsafe.Pointer, sig int, pid int) error {
func (pal *enclaveRuntimePalApiV1) kill(sig int, pid int) error {
sigNum := C.int(sig)
pidNum := C.int(pid)
sym := nsenter.SymAddrPalKill()
ret := C.palKillV1(sym, sigNum, pidNum)
if ret < 0 {
return fmt.Errorf("pal kill() failed with %d", ret)
......@@ -126,9 +141,10 @@ func (pal *enclaveRuntimePalApiV1) kill(sym unsafe.Pointer, sig int, pid int) er
return nil
}
func (pal *enclaveRuntimePalApiV1) destroy(sym unsafe.Pointer) error {
func (pal *enclaveRuntimePalApiV1) destroy() error {
logrus.Debugf("pal destroy() called")
sym := nsenter.SymAddrPalDestroy()
ret := C.palDestroyV1(sym)
if ret < 0 {
return fmt.Errorf("pal destroy() failed with %d", ret)
......
......@@ -2,17 +2,11 @@ package enclave_runtime_pal // import "github.com/opencontainers/runc/libenclave
import (
"github.com/opencontainers/runc/libenclave/configs"
"unsafe"
)
type enclaveRuntimePal struct {
handle unsafe.Pointer
name string
version uint32
init unsafe.Pointer
exec unsafe.Pointer
kill unsafe.Pointer
destroy unsafe.Pointer
}
func StartInitialization(config *configs.InitEnclaveConfig) (*enclaveRuntimePal, error) {
......
package enclave_runtime_pal // import "github.com/opencontainers/runc/libenclave/internal/runtime/pal"
// #cgo LDFLAGS: -ldl
// #define _GNU_SOURCE
// #include <stdlib.h>
// #include <dlfcn.h>
import "C"
import (
"fmt"
"os"
"path"
"strings"
"unsafe"
)
const (
......@@ -29,106 +22,22 @@ func (pal *enclaveRuntimePal) Load(palPath string) (err error) {
}
palName := strings.TrimSuffix(strings.TrimPrefix(bp, palPrefix), palSuffix)
p := C.CString(palPath)
defer C.free(unsafe.Pointer(p))
handle := C.dlmopen(C.LM_ID_NEWLM, p, C.RTLD_LAZY)
if handle == nil {
return fmt.Errorf("unable to load pal %s\n", palPath)
}
defer func() {
if err != nil {
C.dlclose(handle)
}
}()
pal.handle = handle
pal.name = palName
if err = pal.getPalApiVersion(); err != nil {
return err
}
return pal.probeApi()
return nil
}
func (pal *enclaveRuntimePal) getPalApiVersion() error {
return pal.getSymbol("version",
func(sym unsafe.Pointer) error {
if sym == nil {
pal.version = 1
} else {
ver := *(*uint32)(sym)
api := &enclaveRuntimePalApiV1{}
ver := api.get_version()
if ver > palApiVersion {
return fmt.Errorf("unsupported pal api version %d", ver)
}
pal.version = ver
}
return nil
},
)
}
func (pal *enclaveRuntimePal) probeApi() (err error) {
err = pal.getSymbol("init",
func(sym unsafe.Pointer) error {
if sym == nil {
return fmt.Errorf("unresolved api interface %s_pal_init", pal.name)
}
pal.init = sym
return nil
},
)
if err != nil {
return err
}
err = pal.getSymbol("exec",
func(sym unsafe.Pointer) error {
if sym == nil {
return fmt.Errorf("unresolved api interface %s_pal_exec", pal.name)
}
pal.exec = sym
return nil
},
)
if err != nil {
return err
}
err = pal.getSymbol("kill",
func(sym unsafe.Pointer) error {
if sym == nil {
if pal.version == 1 {
return nil
}
return fmt.Errorf("unresolved api interface %s_pal_kill", pal.name)
}
pal.kill = sym
return nil
},
)
if err != nil {
return err
}
err = pal.getSymbol("destroy",
func(sym unsafe.Pointer) error {
if sym == nil {
return fmt.Errorf("unresolved api interface %s_pal_destroy", pal.name)
}
pal.destroy = sym
return nil
},
)
return err
}
func (pal *enclaveRuntimePal) getSymbol(apiName string, handler func(sym unsafe.Pointer) error) error {
symName := fmt.Sprintf("%s_pal_%s", pal.name, apiName)
sn := C.CString(symName)
defer C.free(unsafe.Pointer(sn))
sym := C.dlsym(pal.handle, sn)
return handler(sym)
}
func (pal *enclaveRuntimePal) Name() string {
......@@ -137,7 +46,7 @@ func (pal *enclaveRuntimePal) Name() string {
func (pal *enclaveRuntimePal) Init(args string, logLevel string) error {
api := &enclaveRuntimePalApiV1{}
return api.init(pal.init, args, logLevel)
return api.init(args, logLevel)
}
func (pal *enclaveRuntimePal) Attest() (err error) {
......@@ -146,18 +55,18 @@ func (pal *enclaveRuntimePal) Attest() (err error) {
func (pal *enclaveRuntimePal) Exec(cmd []string, envp []string, stdio [3]*os.File) (int32, error) {
api := &enclaveRuntimePalApiV1{}
return api.exec(pal.exec, cmd, envp, stdio)
return api.exec(cmd, envp, stdio)
}
func (pal *enclaveRuntimePal) Kill(sig int, pid int) error {
if pal.version >= 2 {
api := &enclaveRuntimePalApiV1{}
return api.kill(pal.kill, sig, pid)
return api.kill(sig, pid)
}
return nil
}
func (pal *enclaveRuntimePal) Destroy() error {
api := &enclaveRuntimePalApiV1{}
return api.destroy(pal.destroy)
return api.destroy()
}
......@@ -119,7 +119,6 @@ func main() {
eventsCommand,
execCommand,
initCommand,
enclaveCommand,
killCommand,
listCommand,
pauseCommand,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册