rustc: Implement stack probes for x86

This commit implements stack probes on x86/x86_64 using the freshly landed
support upstream in LLVM. The purpose of stack probes here are to guarantee a
segfault on stack overflow rather than having a chance of running over the guard
page already present on all threads by accident.

At this time there's no support for any other architecture because LLVM itself
does not have support for other architectures.

libcompiler_builtins @ e9b258bc

-Subproject commit 238647af806470dc73e585c03682083931d29cd5
+Subproject commit e9b258bc0cde0f7ed4e36ca96a25d7575e1e58b0

src/librustc_back/target/i386_apple_ios.rs

@@ -26,6 +26,7 @@ pub fn target() -> TargetResult {
         linker_flavor: LinkerFlavor::Gcc,
         options: TargetOptions {
             max_atomic_width: Some(64),
+            stack_probes: true,
             .. base
         }
     })

src/librustc_back/target/i686_apple_darwin.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "yonah".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m32".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-apple-darwin".to_string(),

src/librustc_back/target/i686_linux_android.rs

@@ -22,6 +22,7 @@ pub fn target() -> TargetResult {
     // http://developer.android.com/ndk/guides/abis.html#x86
     base.cpu = "pentiumpro".to_string();
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3".to_string();
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-linux-android".to_string(),

src/librustc_back/target/i686_unknown_dragonfly.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-dragonfly".to_string(),

src/librustc_back/target/i686_unknown_freebsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-freebsd".to_string(),

src/librustc_back/target/i686_unknown_haiku.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m32".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-haiku".to_string(),

src/librustc_back/target/i686_unknown_linux_gnu.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-linux-gnu".to_string(),

src/librustc_back/target/i686_unknown_linux_musl.rs

@@ -17,6 +17,7 @@ pub fn target() -> TargetResult {
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-Wl,-melf_i386".to_string());
+    base.stack_probes = true;
 
     // The unwinder used by i686-unknown-linux-musl, the LLVM libunwind
     // implementation, apparently relies on frame pointers existing... somehow.

src/librustc_back/target/i686_unknown_netbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-netbsdelf".to_string(),

src/librustc_back/target/i686_unknown_openbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "pentium4".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m32".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "i686-unknown-openbsd".to_string(),

src/librustc_back/target/mod.rs

@@ -409,6 +409,9 @@ pub struct TargetOptions {
 
     /// Whether or not the CRT is statically linked by default.
     pub crt_static_default: bool,
+
+    /// Whether or not stack probes (__rust_probestack) are enabled
+    pub stack_probes: bool,
 }
 
 impl Default for TargetOptions {
@@ -466,6 +469,7 @@ fn default() -> TargetOptions {
             panic_strategy: PanicStrategy::Unwind,
             abi_blacklist: vec![],
             crt_static_default: false,
+            stack_probes: false,
         }
     }
 }
@@ -688,6 +692,7 @@ pub fn from_json(obj: Json) -> TargetResult {
         key!(min_atomic_width, Option<u64>);
         try!(key!(panic_strategy, PanicStrategy));
         key!(crt_static_default, bool);
+        key!(stack_probes, bool);
 
         if let Some(array) = obj.find("abi-blacklist").and_then(Json::as_array) {
             for name in array.iter().filter_map(|abi| abi.as_string()) {
@@ -874,6 +879,7 @@ fn to_json(&self) -> Json {
         target_option_val!(max_atomic_width);
         target_option_val!(panic_strategy);
         target_option_val!(crt_static_default);
+        target_option_val!(stack_probes);
 
         if default.abi_blacklist != self.options.abi_blacklist {
             d.insert("abi-blacklist".to_string(), self.options.abi_blacklist.iter()

src/librustc_back/target/x86_64_apple_darwin.rs

@@ -17,6 +17,7 @@ pub fn target() -> TargetResult {
     base.max_atomic_width = Some(128); // core2 support cmpxchg16b
     base.eliminate_frame_pointer = false;
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-apple-darwin".to_string(),

src/librustc_back/target/x86_64_apple_ios.rs

@@ -26,6 +26,7 @@ pub fn target() -> TargetResult {
         linker_flavor: LinkerFlavor::Gcc,
         options: TargetOptions {
             max_atomic_width: Some(64),
+            stack_probes: true,
             .. base
         }
     })

src/librustc_back/target/x86_64_linux_android.rs

@@ -18,6 +18,7 @@ pub fn target() -> TargetResult {
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-linux-android".to_string(),

src/librustc_back/target/x86_64_rumprun_netbsd.rs

@@ -25,6 +25,7 @@ pub fn target() -> TargetResult {
     base.disable_redzone = true;
     base.no_default_libraries = false;
     base.exe_allocation_crate = None;
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-rumprun-netbsd".to_string(),

src/librustc_back/target/x86_64_sun_solaris.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-pc-solaris".to_string(),

src/librustc_back/target/x86_64_unknown_bitrig.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-bitrig".to_string(),

src/librustc_back/target/x86_64_unknown_dragonfly.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-dragonfly".to_string(),

src/librustc_back/target/x86_64_unknown_freebsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-freebsd".to_string(),

src/librustc_back/target/x86_64_unknown_fuchsia.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-fuchsia".to_string(),

src/librustc_back/target/x86_64_unknown_haiku.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-m64".to_string()]);
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-haiku".to_string(),

src/librustc_back/target/x86_64_unknown_linux_gnu.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-linux-gnu".to_string(),

src/librustc_back/target/x86_64_unknown_linux_musl.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-linux-musl".to_string(),

src/librustc_back/target/x86_64_unknown_netbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-netbsd".to_string(),

src/librustc_back/target/x86_64_unknown_openbsd.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-openbsd".to_string(),

src/librustc_back/target/x86_64_unknown_redox.rs

@@ -16,6 +16,7 @@ pub fn target() -> TargetResult {
     base.cpu = "x86-64".to_string();
     base.max_atomic_width = Some(64);
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-m64".to_string());
+    base.stack_probes = true;
 
     Ok(Target {
         llvm_target: "x86_64-unknown-redox".to_string(),

src/librustc_trans/attributes.rs

@@ -11,13 +11,14 @@
 
 use std::ffi::{CStr, CString};
 
+use rustc::session::config::Sanitizer;
+
 use llvm::{self, Attribute, ValueRef};
 use llvm::AttributePlace::Function;
 pub use syntax::attr::{self, InlineAttr};
 use syntax::ast;
 use context::CrateContext;
 
-
 /// Mark LLVM function to use provided inline heuristic.
 #[inline]
 pub fn inline(val: ValueRef, inline: InlineAttr) {
@@ -69,6 +70,28 @@ pub fn set_frame_pointer_elimination(ccx: &CrateContext, llfn: ValueRef) {
     }
 }
 
+pub fn set_probestack(ccx: &CrateContext, llfn: ValueRef) {
+    // Only use stack probes if the target specification indicates that we
+    // should be using stack probes
+    if !ccx.sess().target.target.options.stack_probes {
+        return
+    }
+
+    // Currently stack probes seem somewhat incompatible with the address
+    // sanitizer. With asan we're already protected from stack overflow anyway
+    // so we don't really need stack probes regardless.
+    match ccx.sess().opts.debugging_opts.sanitizer {
+        Some(Sanitizer::Address) => return,
+        _ => {}
+    }
+
+    // Flag our internal `__rust_probestack` function as the stack probe symbol.
+    // This is defined in the `compiler-builtins` crate for each architecture.
+    llvm::AddFunctionAttrStringValue(
+        llfn, llvm::AttributePlace::Function,
+        cstr("probe-stack\0"), cstr("__rust_probestack\0"));
+}
+
 /// Composite function which sets LLVM attributes for function depending on its AST (#[attribute])
 /// attributes.
 pub fn from_fn_attrs(ccx: &CrateContext, attrs: &[ast::Attribute], llfn: ValueRef) {
@@ -76,6 +99,7 @@ pub fn from_fn_attrs(ccx: &CrateContext, attrs: &[ast::Attribute], llfn: ValueRe
     inline(llfn, find_inline_attr(Some(ccx.sess().diagnostic()), attrs));
 
     set_frame_pointer_elimination(ccx, llfn);
+    set_probestack(ccx, llfn);
     let mut target_features = vec![];
     for attr in attrs {
         if attr.check_name("target_feature") {

llvm @ f0a23af5

-Subproject commit 8e1b4fedfa4542e65f82fe124bd6433f3bd0aec5
+Subproject commit f0a23af57a17658e75d6dc50568397590a0e6664

src/test/codegen/stack-probes.rs

+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// ignore-arm
+// ignore-wasm
+// ignore-emscripten
+// ignore-windows
+// no-system-llvm
+// compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn foo() {
+// CHECK: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}}"probe-stack"="__rust_probestack"{{.*}} }
+}

src/test/run-pass/stack-probes-lto.rs

+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// ignore-arm
+// ignore-wasm
+// ignore-emscripten
+// ignore-musl FIXME #31506
+// ignore-pretty
+// no-system-llvm
+// compile-flags: -C lto
+// no-prefer-dynamic
+
+include!("stack-probes.rs");

src/test/run-pass/stack-probes.rs

+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// ignore-arm
+// ignore-wasm
+// ignore-emscripten
+// ignore-musl FIXME #31506
+// no-system-llvm
+
+use std::mem;
+use std::process::Command;
+use std::thread;
+use std::env;
+
+#[link(name = "rust_test_helpers", kind = "static")]
+extern {
+    #[link_name = "rust_dbg_extern_identity_u64"]
+    fn black_box(u: u64);
+}
+
+fn main() {
+    let args = env::args().skip(1).collect::<Vec<_>>();
+    if args.len() > 0 {
+        match &args[0][..] {
+            "main-thread" => recurse(&[]),
+            "child-thread" => thread::spawn(|| recurse(&[])).join().unwrap(),
+            _ => panic!(),
+        }
+        return
+    }
+
+    let me = env::current_exe().unwrap();
+
+    // The linux kernel has some different behavior for the main thread because
+    // the main thread's stack can typically grow. We can't always guarantee
+    // that we report stack overflow on the main thread, see #43052 for some
+    // details
+    if cfg!(not(target_os = "linux")) {
+        assert_overflow(Command::new(&me).arg("main-thread"));
+    }
+    assert_overflow(Command::new(&me).arg("child-thread"));
+}
+
+#[allow(unconditional_recursion)]
+fn recurse(array: &[u64]) {
+    unsafe { black_box(array.as_ptr() as u64); }
+    let local: [_; 1024] = unsafe { mem::uninitialized() };
+    recurse(&local);
+}
+
+fn assert_overflow(cmd: &mut Command) {
+    let output = cmd.output().unwrap();
+    assert!(!output.status.success());
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    println!("status: {}", output.status);
+    println!("stdout: {}", stdout);
+    println!("stderr: {}", stderr);
+    assert!(stdout.is_empty());
+    assert!(stderr.contains("has overflowed its stack\n"));
+}