Merge branch 'master' into features/shim-makefile

README.md

@@ -64,5 +64,13 @@ For more information about Enclave Runtime PAL API, please refer to [Enclave Run
 ### Run OCI bundle
 Please refer to [this guide](https://github.com/alibaba/inclavare-containers/blob/master/docs/running_rune_with_occlum_bundle.md) to run `occlum bundle` with `rune`.
 
+---
+
 ## Run rune containers in Kubernetes cluster
-Please refer to [this guide](docs/develop_and_deploy_hello_world_application_in_kubernetes_cluster.md) to develop and deploy a rune container in a Kubernetes cluster.
\ No newline at end of file
+Please refer to [this guide](docs/develop_and_deploy_hello_world_application_in_kubernetes_cluster.md) to develop and deploy a rune container in a Kubernetes cluster.
+
+---
+
+## Reference container image
+[The reference container images](https://hub.docker.com/u/inclavarecontainers) are available for the demonstration purpose to show how a Confidential Computing Kubernetes Cluster with Inclavare Containers works. Currently, web application demos based on OpenJDK 11 and Golang are provided.
+

patch/no-sgx-flc/0001-psw-Support-SGX1-machine-with-SGX-in-tree-driver.patch

+From 3d8cb6ba2017d03282c25d8f5126418f456203d2 Mon Sep 17 00:00:00 2001
+From: Jia Zhang <zhang.jia@linux.alibaba.com>
+Date: Wed, 29 Jul 2020 11:20:40 +0800
+Subject: [PATCH] psw: Support SGX1 machine with SGX in-tree driver
+
+There are still lots of SGX1 machines without FLC support deployed
+in filed. These machines eventually needs to be migrated to be supported
+by SGX in-tree driver which is product-ready and well-maintained.
+
+This patch targets to address the gap between SGX1 machine and SGX
+in-tree driver. This patch is incomplete. In theory, reading cpuid,
+msr and driver type probe can decide whether SGX1 without FLC support
+and SGX in-tree driver are placed together and internally set
+no_sgx_flc flag.
+
+Signed-off-by: Jia Zhang <zhang.jia@linux.alibaba.com>
+Signed-off-by: Liang Yang <liang3.yang@intel.com>
+---
+ psw/enclave_common/sgx_enclave_common.cpp |  8 ++++++--
+ psw/urts/enclave_creator_hw_com.cpp       | 10 +++++++++-
+ psw/urts/linux/urts_internal.cpp          |  3 ++-
+ 3 files changed, 17 insertions(+), 4 deletions(-)
+
+diff --git a/psw/enclave_common/sgx_enclave_common.cpp b/psw/enclave_common/sgx_enclave_common.cpp
+index 8131223..e4394b0 100644
+--- a/psw/enclave_common/sgx_enclave_common.cpp
++++ b/psw/enclave_common/sgx_enclave_common.cpp
+@@ -52,6 +52,7 @@
+ #define SGX_GET_LAUNCH_TOKEN "get_launch_token"
+ 
+ func_get_launch_token_t get_launch_token_func = NULL;
++bool no_sgx_flc = true;
+ 
+ static void* s_hdlopen = NULL;
+ static se_mutex_t s_dlopen_mutex;
+@@ -807,7 +808,7 @@ extern "C" bool COMM_API enclave_initialize(
+     }
+ 
+     int ret = 0;
+-    if ( s_driver_type == SGX_DRIVER_OUT_OF_TREE )
++    if ( s_driver_type == SGX_DRIVER_OUT_OF_TREE || no_sgx_flc)
+     {
+         //out-of-tree driver requires a launch token to be provided
+         se_mutex_lock(&s_enclave_mutex);
+@@ -846,7 +847,10 @@ extern "C" bool COMM_API enclave_initialize(
+         initp.sigstruct = POINTER_TO_U64(enclave_css);
+         initp.einittoken = POINTER_TO_U64(&launch_token);
+ 
+-        ret = ioctl(s_hdevice, SGX_IOC_ENCLAVE_INIT, &initp);
++        if (hfile != -1)
++            ret = ioctl(hfile, SGX_IOC_ENCLAVE_INIT, &initp);
++        else
++            ret = ioctl(s_hdevice, SGX_IOC_ENCLAVE_INIT, &initp);
+     } 
+     else if (s_driver_type == SGX_DRIVER_DCAP )
+     {
+diff --git a/psw/urts/enclave_creator_hw_com.cpp b/psw/urts/enclave_creator_hw_com.cpp
+index 28d10a4..3d6c760 100644
+--- a/psw/urts/enclave_creator_hw_com.cpp
++++ b/psw/urts/enclave_creator_hw_com.cpp
+@@ -43,6 +43,9 @@
+ 
+ #define EDMM_ENABLE_BIT 0x1ULL
+ 
++extern "C" bool get_driver_type(int *driver_type);
++extern bool no_sgx_flc;
++
+ bool EnclaveCreatorHW::use_se_hw() const
+ {
+     return true;
+@@ -164,6 +167,10 @@ int EnclaveCreatorHW::init_enclave(sgx_enclave_id_t enclave_id, enclave_css_t *e
+     UNUSED(lc);
+ 
+     unsigned int ret = 0;
++    int driver_type;
++
++    if (!get_driver_type(&driver_type))
++        return SGX_ERROR_UNEXPECTED;
+ 
+     enclave_css_t css;
+     memcpy_s(&css, sizeof(enclave_css_t), enclave_css, sizeof(enclave_css_t));
+@@ -178,7 +185,8 @@ int EnclaveCreatorHW::init_enclave(sgx_enclave_id_t enclave_id, enclave_css_t *e
+         {
+             // LE is loaded with the interface sgx_create_le.
+             // Read the input prd css file and use it to init again.
+-            if(SGX_ERROR_INVALID_ATTRIBUTE == ret && prd_css_file != NULL) {
++            // Note that in-tree kernel driver never expose SGX_ERROR_INVALID_ATTRIBUTE.
++            if((SGX_ERROR_INVALID_ATTRIBUTE == ret || (no_sgx_flc && driver_type == SGX_DRIVER_IN_KERNEL)) && prd_css_file != NULL) {
+                 if((ret = read_prd_css(prd_css_file->prd_css_name, &css)) != SGX_SUCCESS)
+                 {
+                     return ret;
+diff --git a/psw/urts/linux/urts_internal.cpp b/psw/urts/linux/urts_internal.cpp
+index 502f7b5..1885830 100644
+--- a/psw/urts/linux/urts_internal.cpp
++++ b/psw/urts/linux/urts_internal.cpp
+@@ -46,6 +46,7 @@
+ extern sgx_status_t _create_enclave(const bool debug, se_file_handle_t pfile, se_file_t& file, le_prd_css_file_t *prd_css_file, sgx_launch_token_t *launch, int *launch_updated, sgx_enclave_id_t *enclave_id, sgx_misc_attribute_t *misc_attr);
+ 
+ extern func_get_launch_token_t get_launch_token_func;
++extern bool no_sgx_flc;
+ 
+ extern "C" void init_get_launch_token(const func_get_launch_token_t func)
+ {
+@@ -90,6 +91,6 @@ extern "C" sgx_status_t sgx_create_le(const char* file_name, const char* prd_css
+ extern "C" bool is_launch_token_required()
+ {
+     //noly out of tree driver need to get launch token
+-    return is_out_of_tree_driver();
++    return no_sgx_flc || is_out_of_tree_driver();
+ }
+ 
+-- 
+1.8.3.1
+

patch/no-sgx-flc/0001-sgx-Support-SGX1-machine-even-without-FLC-support.patch

+From 00cfba73fce954b7e5e8da24fccae48537a0af49 Mon Sep 17 00:00:00 2001
+From: Jia Zhang <zhang.jia@linux.alibaba.com>
+Date: Sun, 26 Jul 2020 19:01:24 +0800
+Subject: [PATCH] sgx: Support SGX1 machine even without FLC support
+
+There are still lots of SGX1 machines without FLC support deployed
+in filed. These machines eventually needs to be migrated to be supported
+by SGX in-tree driver which is product-ready and well-maintained.
+
+This patch targets to address the gap between SGX1 machine and SGX
+in-tree driver.
+
+Signed-off-by: Jia Zhang <zhang.jia@linux.alibaba.com>
+---
+ arch/x86/include/uapi/asm/sgx.h  | 15 ++++++++++
+ arch/x86/kernel/cpu/feat_ctl.c   | 15 ++++------
+ arch/x86/kernel/cpu/sgx/driver.c |  5 ----
+ arch/x86/kernel/cpu/sgx/ioctl.c  | 63 ++++++++++++++++++++++++++++++++++++++++
+ 4 files changed, 84 insertions(+), 14 deletions(-)
+
+diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h
+index 3760e5d..df24dd9 100644
+--- a/arch/x86/include/uapi/asm/sgx.h
++++ b/arch/x86/include/uapi/asm/sgx.h
+@@ -25,6 +25,8 @@ enum sgx_page_flags {
+ 	_IOWR(SGX_MAGIC, 0x01, struct sgx_enclave_add_pages)
+ #define SGX_IOC_ENCLAVE_INIT \
+ 	_IOW(SGX_MAGIC, 0x02, struct sgx_enclave_init)
++#define SGX_IOC_ENCLAVE_INIT_WITH_TOKEN \
++	_IOW(SGX_MAGIC, 0x02, struct sgx_enclave_init_with_token)
+ #define SGX_IOC_ENCLAVE_SET_ATTRIBUTE \
+ 	_IOW(SGX_MAGIC, 0x03, struct sgx_enclave_set_attribute)
+ 
+@@ -66,6 +68,19 @@ struct sgx_enclave_init {
+ };
+ 
+ /**
++ * struct sgx_enclave_init_with_token - parameter structure for the
++ *                                      %SGX_IOC_ENCLAVE_INIT_WITH_TOKEN ioctl
++ * @addr:       address in the ELRANGE
++ * @sigstruct:  address for the SIGSTRUCT data
++ * @einittoken: address for the EINITTOKEN data
++ */
++struct sgx_enclave_init_with_token {
++	__u64 addr;
++	__u64 sigstruct;
++	__u64 einittoken;
++} __packed;
++
++/**
+  * struct sgx_enclave_set_attribute - parameter structure for the
+  *				      %SGX_IOC_ENCLAVE_SET_ATTRIBUTE ioctl
+  * @attribute_fd:	file handle of the attribute file in the securityfs
+diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c
+index 1837df3..dad2f1b 100644
+--- a/arch/x86/kernel/cpu/feat_ctl.c
++++ b/arch/x86/kernel/cpu/feat_ctl.c
+@@ -122,13 +122,8 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c)
+ 		return;
+ 	}
+ 
+-	/*
+-	 * Enable SGX if and only if the kernel supports SGX and Launch Control
+-	 * is supported, i.e. disable SGX if the LE hash MSRs can't be written.
+-	 */
+ 	enable_sgx = cpu_has(c, X86_FEATURE_SGX) &&
+ 		     cpu_has(c, X86_FEATURE_SGX1) &&
+-		     cpu_has(c, X86_FEATURE_SGX_LC) &&
+ 		     IS_ENABLED(CONFIG_INTEL_SGX);
+ 
+ 	if (msr & FEAT_CTL_LOCKED)
+@@ -152,8 +147,11 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c)
+ 			msr |= FEAT_CTL_VMX_ENABLED_INSIDE_SMX;
+ 	}
+ 
+-	if (enable_sgx)
+-		msr |= FEAT_CTL_SGX_ENABLED | FEAT_CTL_SGX_LC_ENABLED;
++	if (enable_sgx) {
++		msr |= FEAT_CTL_SGX_ENABLED;
++		if (cpu_has(c, X86_FEATURE_SGX_LC))
++			msr |= FEAT_CTL_SGX_LC_ENABLED;
++	}
+ 
+ 	wrmsrl(MSR_IA32_FEAT_CTL, msr);
+ 
+@@ -176,8 +174,7 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c)
+ 	}
+ 
+ update_sgx:
+-	if (!(msr & FEAT_CTL_SGX_ENABLED) ||
+-	    !(msr & FEAT_CTL_SGX_LC_ENABLED) || !enable_sgx) {
++	if (!(msr & FEAT_CTL_SGX_ENABLED) || !enable_sgx) {
+ 		if (enable_sgx)
+ 			pr_err_once("SGX disabled by BIOS\n");
+ 		clear_sgx_caps();
+diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c
+index 997a7f4..4397d72 100644
+--- a/arch/x86/kernel/cpu/sgx/driver.c
++++ b/arch/x86/kernel/cpu/sgx/driver.c
+@@ -165,11 +165,6 @@ int __init sgx_drv_init(void)
+ 	int ret;
+ 	int i;
+ 
+-	if (!boot_cpu_has(X86_FEATURE_SGX_LC)) {
+-		pr_info("The public key MSRs are not writable.\n");
+-		return -ENODEV;
+-	}
+-
+ 	cpuid_count(SGX_CPUID, 0, &eax, &ebx, &ecx, &edx);
+ 	sgx_misc_reserved_mask = ~ebx | SGX_MISC_RESERVED_MASK;
+ 	sgx_encl_size_max_64 = 1ULL << ((edx >> 8) & 0xFF);
+diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c
+index 9227ba9..0eb79b7 100644
+--- a/arch/x86/kernel/cpu/sgx/ioctl.c
++++ b/arch/x86/kernel/cpu/sgx/ioctl.c
+@@ -209,6 +209,8 @@ static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs)
+ 	encl->secs.encl = encl;
+ 	encl->secs_attributes = secs->attributes;
+ 	encl->allowed_attributes |= SGX_ATTR_ALLOWED_MASK;
++	if (!boot_cpu_has(X86_FEATURE_SGX_LC))
++		encl->allowed_attributes |= SGX_ATTR_EINITTOKENKEY;
+ 	encl->base = secs->base;
+ 	encl->size = secs->size;
+ 	encl->ssaframesize = secs->ssa_frame_size;
+@@ -694,6 +696,11 @@ static long sgx_ioc_enclave_init(struct sgx_encl *encl, void __user *arg)
+ 	void *token;
+ 	int ret;
+ 
++	if (!boot_cpu_has(X86_FEATURE_SGX_LC)) {
++		pr_info("The public key MSRs are not writable.\n");
++		return -ENODEV;
++	}
++
+ 	if (!(atomic_read(&encl->flags) & SGX_ENCL_CREATED))
+ 		return -EINVAL;
+ 
+@@ -735,6 +742,59 @@ static long sgx_ioc_enclave_init(struct sgx_encl *encl, void __user *arg)
+ 	return ret;
+ }
+ 
++static long sgx_ioc_enclave_init_with_token(struct sgx_encl *encl, void __user *arg)
++{
++	struct sgx_sigstruct *sigstruct;
++	struct sgx_enclave_init_with_token einit;
++	struct page *initp_page;
++	void *token;
++	int ret;
++
++	if (!(atomic_read(&encl->flags) & SGX_ENCL_CREATED))
++		return -EINVAL;
++
++	if (copy_from_user(&einit, arg, sizeof(einit)))
++		return -EFAULT;
++
++	initp_page = alloc_page(GFP_KERNEL);
++	if (!initp_page)
++		return -ENOMEM;
++
++	sigstruct = kmap(initp_page);
++	token = (void *)((unsigned long)sigstruct + PAGE_SIZE / 2);
++
++	if (copy_from_user(token, (void __user *)einit.einittoken,
++			   SGX_LAUNCH_TOKEN_SIZE)) {
++		ret = -EFAULT;
++		goto out;
++	}
++
++	if (copy_from_user(sigstruct, (void __user *)einit.sigstruct,
++			   sizeof(*sigstruct))) {
++		ret = -EFAULT;
++		goto out;
++	}
++
++	/*
++	 * A legacy field used with Intel signed enclaves. These used to mean
++	 * regular and architectural enclaves. The CPU only accepts these values
++	 * but they do not have any other meaning.
++	 *
++	 * Thus, reject any other values.
++	 */
++	if (sigstruct->header.vendor != 0x0000 &&
++	    sigstruct->header.vendor != 0x8086) {
++		ret = -EINVAL;
++		goto out;
++	}
++
++	ret = sgx_encl_init(encl, sigstruct, token);
++out:
++	kunmap(initp_page);
++	__free_page(initp_page);
++	return ret;
++}
++
+ /**
+  * sgx_ioc_enclave_set_attribute - handler for %SGX_IOC_ENCLAVE_SET_ATTRIBUTE
+  * @filep:	open file to /dev/sgx
+@@ -804,6 +864,9 @@ long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
+ 	case SGX_IOC_ENCLAVE_INIT:
+ 		ret = sgx_ioc_enclave_init(encl, (void __user *)arg);
+ 		break;
++	case SGX_IOC_ENCLAVE_INIT_WITH_TOKEN:
++		ret = sgx_ioc_enclave_init_with_token(encl, (void __user *)arg);
++		break;
+ 	case SGX_IOC_ENCLAVE_SET_ATTRIBUTE:
+ 		ret = sgx_ioc_enclave_set_attribute(encl, (void __user *)arg);
+ 		break;
+-- 
+1.8.3.1
+

patch/no-sgx-flc/README.md

+There are still non-trivial number of systems without FLC support.
+
+# Prerequisite
+- Apply the patch `0001-sgx-Support-SGX1-machine-even-without-FLC-support.patch` to [v33 SGX in-tree driver](https://github.com/jsakkine-intel/linux-sgx/tree/v33).
+- Apply the patch `0001-psw-Support-SGX1-machine-with-SGX-in-tree-driver.patch` to [Intel SGX SDK 2.10](https://github.com/intel/linux-sgx/tree/sgx_2.10) or higher.
+
+# Validation
+- Successfully run [sgx-tools](https://github.com/alibaba/inclavare-containers/tree/master/sgx-tools#test) to generate a launch token.
+- Successfully run [skeleton bundle](https://github.com/alibaba/inclavare-containers/blob/master/rune/libenclave/internal/runtime/pal/skeleton/README.md).
+  * Note: specify `"enclave.runtime.args": "no-sgx-flc"` in config.json is required.