rust_trans: struct argument over ffi were passed incorrectly in some situations on x86_64.

src/librustc_trans/trans/cabi_x86_64.rs

@@ -410,18 +410,53 @@ fn x86_64_ty<F>(ccx: &CrateContext,
         }
     }
 
-    let mut arg_tys = Vec::new();
-    for t in atys {
-        let ty = x86_64_ty(ccx, *t, |cls| cls.is_pass_byval(), Attribute::ByVal);
-        arg_tys.push(ty);
-    }
+    let mut int_regs = 6; // RDI, RSI, RDX, RCX, R8, R9
+    let mut sse_regs = 8;
 
     let ret_ty = if ret_def {
-        x86_64_ty(ccx, rty, |cls| cls.is_ret_bysret(), Attribute::StructRet)
+        x86_64_ty(ccx, rty, |cls| {
+            if cls.is_ret_bysret() {
+                // `sret` parameter thus one less register available
+                int_regs -= 1;
+                true
+            } else {
+                false
+            }
+        }, Attribute::StructRet)
     } else {
         ArgType::direct(Type::void(ccx), None, None, None)
     };
 
+    let mut arg_tys = Vec::new();
+    for t in atys {
+        let ty = x86_64_ty(ccx, *t, |cls| {
+            let needed_int = cls.iter().filter(|&&c| c == Int).count();
+            let needed_sse = cls.iter().filter(|c| c.is_sse()).count();
+            let in_mem = cls.is_pass_byval() ||
+                         int_regs < needed_int ||
+                         sse_regs < needed_sse;
+            if in_mem {
+                // `byval` parameter thus one less integer register available
+                int_regs -= 1;
+            } else {
+                // split into sized chunks passed individually
+                int_regs -= needed_int;
+                sse_regs -= needed_sse;
+            }
+            in_mem
+        }, Attribute::ByVal);
+        arg_tys.push(ty);
+
+        // An integer, pointer, double or float parameter
+        // thus the above closure passed to `x86_64_ty` won't
+        // get called.
+        if t.kind() == Integer || t.kind() == Pointer {
+            int_regs -= 1;
+        } else if t.kind() == Double || t.kind() == Float {
+            sse_regs -= 1;
+        }
+    }
+
     return FnType {
         arg_tys: arg_tys,
         ret_ty: ret_ty,

src/test/run-make/extern-fn-struct-passing-abi/Makefile

+-include ../tools.mk
+
+all: $(call NATIVE_STATICLIB,test)
+	$(RUSTC) test.rs
+	$(call RUN,test) || exit 1

src/test/run-make/extern-fn-struct-passing-abi/test.c

+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#include <assert.h>
+#include <stdint.h>
+
+struct Rect {
+    int32_t a;
+    int32_t b;
+    int32_t c;
+    int32_t d;
+};
+
+struct BiggerRect {
+    struct Rect s;
+    int32_t a;
+    int32_t b;
+};
+
+struct FloatRect {
+    int32_t a;
+    int32_t b;
+    double c;
+};
+
+struct Huge {
+    int32_t a;
+    int32_t b;
+    int32_t c;
+    int32_t d;
+    int32_t e;
+};
+
+// SysV ABI:
+// a, b, c, d, e should be in registers
+// s should be byval pointer
+void byval_rect(int32_t a, int32_t b, int32_t c, int32_t d, int32_t e, struct Rect s) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(c == 3);
+    assert(d == 4);
+    assert(e == 5);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+}
+
+// SysV ABI:
+// a, b, c, d, e, f, g should be in sse registers
+// s should be split across 2 registers
+// t should be byval pointer
+void byval_rect_floats(float a, float b, double c, float d, float e,
+                       float f, double g, struct Rect s, struct FloatRect t) {
+    assert(a == 1.);
+    assert(b == 2.);
+    assert(c == 3.);
+    assert(d == 4.);
+    assert(e == 5.);
+    assert(f == 6.);
+    assert(g == 7.);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+    assert(t.a == 3489);
+    assert(t.b == 3490);
+    assert(t.c == 8.);
+}
+
+// SysV ABI:
+// a, b, d, e should be in registers
+// c passed via sse registers
+// s should be byval pointer
+void byval_rect_with_float(int32_t a, int32_t b, float c, int32_t d,
+                           int32_t e, int32_t f, struct Rect s) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(c == 3.);
+    assert(d == 4);
+    assert(e == 5);
+    assert(f == 6);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+}
+
+// SysV ABI:
+// a, b should be in registers
+// s should be split across 2 registers
+void split_rect(int32_t a, int32_t b, struct Rect s) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+}
+
+// SysV ABI:
+// a, b should be in sse registers
+// s should be split across int32_t & sse registers
+void split_rect_floats(float a, float b, struct FloatRect s) {
+    assert(a == 1.);
+    assert(b == 2.);
+    assert(s.a == 3489);
+    assert(s.b == 3490);
+    assert(s.c == 8.);
+}
+
+// SysV ABI:
+// a, b, d, f should be in registers
+// c, e passed via sse registers
+// s should be split across 2 registers
+void split_rect_with_floats(int32_t a, int32_t b, float c,
+                            int32_t d, float e, int32_t f, struct Rect s) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(c == 3.);
+    assert(d == 4);
+    assert(e == 5.);
+    assert(f == 6);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+}
+
+// SysV ABI:
+// a, b, c should be in registers
+// s should be split across 2 registers
+// t should be a byval pointer
+void split_and_byval_rect(int32_t a, int32_t b, int32_t c, struct Rect s, struct Rect t) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(c == 3);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+    assert(t.a == 553);
+    assert(t.b == 554);
+    assert(t.c == 555);
+    assert(t.d == 556);
+}
+
+// SysV ABI:
+// a, b should in registers
+// s and return should be split across 2 registers
+struct Rect split_ret_byval_struct(int32_t a, int32_t b, struct Rect s) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+    return s;
+}
+
+// SysV ABI:
+// a, b, c, d should be in registers
+// return should be in a hidden sret pointer
+// s should be a byval pointer
+struct BiggerRect sret_byval_struct(int32_t a, int32_t b, int32_t c, int32_t d, struct Rect s) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(c == 3);
+    assert(d == 4);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+
+    struct BiggerRect t;
+    t.s = s; t.a = 27834; t.b = 7657;
+    return t;
+}
+
+// SysV ABI:
+// a, b should be in registers
+// return should be in a hidden sret pointer
+// s should be split across 2 registers
+struct BiggerRect sret_split_struct(int32_t a, int32_t b, struct Rect s) {
+    assert(a == 1);
+    assert(b == 2);
+    assert(s.a == 553);
+    assert(s.b == 554);
+    assert(s.c == 555);
+    assert(s.d == 556);
+
+    struct BiggerRect t;
+    t.s = s; t.a = 27834; t.b = 7657;
+    return t;
+}
+
+// SysV ABI:
+// s should be byval pointer (since sizeof(s) > 16)
+// return should in a hidden sret pointer
+struct Huge huge_struct(struct Huge s) {
+    assert(s.a == 5647);
+    assert(s.b == 5648);
+    assert(s.c == 5649);
+    assert(s.d == 5650);
+    assert(s.e == 5651);
+
+    return s;
+}

src/test/run-make/extern-fn-struct-passing-abi/test.rs

+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Passing structs via FFI should work regardless of whether
+// the functions gets passed in multiple registers or is a hidden pointer
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+#[repr(C)]
+struct Rect {
+    a: i32,
+    b: i32,
+    c: i32,
+    d: i32
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+#[repr(C)]
+struct BiggerRect {
+    s: Rect,
+    a: i32,
+    b: i32
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+#[repr(C)]
+struct FloatRect {
+    a: i32,
+    b: i32,
+    c: f64
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+#[repr(C)]
+struct Huge {
+    a: i32,
+    b: i32,
+    c: i32,
+    d: i32,
+    e: i32
+}
+
+#[link(name = "test", kind = "static")]
+extern {
+    // SysV ABI:
+    // a, b, c, d, e should be in registers
+    // s should be byval pointer
+    fn byval_rect(a: i32, b: i32, c: i32, d: i32, e: i32, s: Rect);
+
+    // SysV ABI:
+    // a, b, c, d, e, f, g should be in sse registers
+    // s should be split across 2 registers
+    // t should be byval pointer
+    fn byval_rect_floats(a: f32, b: f32, c: f64, d: f32, e: f32,
+                         f: f32, g: f64, s: Rect, t: FloatRect);
+
+    // SysV ABI:
+    // a, b, d, e should be in registers
+    // c passed via sse registers
+    // s should be byval pointer
+    fn byval_rect_with_float(a: i32, b: i32, c: f32, d: i32, e: i32, f: i32, s: Rect);
+
+    // SysV ABI:
+    // a, b should be in registers
+    // s should be split across 2 registers
+    fn split_rect(a: i32, b: i32, s: Rect);
+
+    // SysV ABI:
+    // a, b should be in sse registers
+    // s should be split across int & sse registers
+    fn split_rect_floats(a: f32, b: f32, s: FloatRect);
+
+    // SysV ABI:
+    // a, b, d, f should be in registers
+    // c, e passed via sse registers
+    // s should be split across 2 registers
+    fn split_rect_with_floats(a: i32, b: i32, c: f32, d: i32, e: f32, f: i32, s: Rect);
+
+    // SysV ABI:
+    // a, b, c should be in registers
+    // s should be split across 2 registers
+    // t should be a byval pointer
+    fn split_and_byval_rect(a: i32, b: i32, c: i32, s: Rect, t: Rect);
+
+    // SysV ABI:
+    // a, b should in registers
+    // s and return should be split across 2 registers
+    fn split_ret_byval_struct(a: i32, b: i32, s: Rect) -> Rect;
+
+    // SysV ABI:
+    // a, b, c, d should be in registers
+    // return should be in a hidden sret pointer
+    // s should be a byval pointer
+    fn sret_byval_struct(a: i32, b: i32, c: i32, d: i32, s: Rect) -> BiggerRect;
+
+    // SysV ABI:
+    // a, b should be in registers
+    // return should be in a hidden sret pointer
+    // s should be split across 2 registers
+    fn sret_split_struct(a: i32, b: i32, s: Rect) -> BiggerRect;
+
+    // SysV ABI:
+    // s should be byval pointer (since sizeof(s) > 16)
+    // return should in a hidden sret pointer
+    fn huge_struct(s: Huge) -> Huge;
+}
+
+fn main() {
+    let s = Rect { a: 553, b: 554, c: 555, d: 556 };
+    let t = BiggerRect { s: s, a: 27834, b: 7657 };
+    let u = FloatRect { a: 3489, b: 3490, c: 8. };
+    let v = Huge { a: 5647, b: 5648, c: 5649, d: 5650, e: 5651 };
+
+    unsafe {
+        byval_rect(1, 2, 3, 4, 5, s);
+        byval_rect_floats(1., 2., 3., 4., 5., 6., 7., s, u);
+        byval_rect_with_float(1, 2, 3.0, 4, 5, 6, s);
+        split_rect(1, 2, s);
+        split_rect_floats(1., 2., u);
+        split_rect_with_floats(1, 2, 3.0, 4, 5.0, 6, s);
+        split_and_byval_rect(1, 2, 3, s, s);
+        split_rect(1, 2, s);
+        assert_eq!(huge_struct(v), v);
+        assert_eq!(split_ret_byval_struct(1, 2, s), s);
+        assert_eq!(sret_byval_struct(1, 2, 3, 4, s), t);
+        assert_eq!(sret_split_struct(1, 2, s), t);
+    }
+}