Enforce object safety

closes #17670

[breaking-change]

Traits must be object-safe if they are to be used in trait objects. This might require splitting a trait into object-safe and non-object-safe parts.

Some standard library traits in std::io have been split - Reader has new traits BytesReader (for the bytes method) and AsRefReader (for by_ref), Writer has new trait AsRefWriter (for by_ref). All these new traits have blanket impls, so any type which implements Reader or Writer (respectively) will have an implmentation of the new traits. To fix your code, you just need to `use` the new trait.

src/librustc/middle/typeck/check/method.rs

@@ -1336,16 +1336,6 @@ fn confirm_candidate(&self, rcvr_ty: ty::t, candidate: &Candidate)
                self.ty_to_string(rcvr_ty),
                candidate.repr(self.tcx()));
 
-        let mut rcvr_substs = candidate.rcvr_substs.clone();
-
-        if !self.enforce_object_limitations(candidate) {
-            // Here we change `Self` from `Trait` to `err` in the case that
-            // this is an illegal object method. This is necessary to prevent
-            // the user from getting strange, derivative errors when the method
-            // takes an argument/return-type of type `Self` etc.
-            rcvr_substs.types.get_mut_slice(SelfSpace)[0] = ty::mk_err();
-        }
-
         self.enforce_drop_trait_limitations(candidate);
 
         // Determine the values for the generic parameters of the method.
@@ -1554,71 +1544,6 @@ fn fixup_derefs_on_method_receiver_if_necessary(
         }
     }
 
-    fn enforce_object_limitations(&self, candidate: &Candidate) -> bool {
-        /*!
-         * There are some limitations to calling functions through an
-         * object, because (a) the self type is not known
-         * (that's the whole point of a trait instance, after all, to
-         * obscure the self type) and (b) the call must go through a
-         * vtable and hence cannot be monomorphized.
-         */
-
-        match candidate.origin {
-            MethodStatic(..) |
-            MethodTypeParam(..) |
-            MethodStaticUnboxedClosure(..) => {
-                return true; // not a call to a trait instance
-            }
-            MethodTraitObject(..) => {}
-        }
-
-        match candidate.method_ty.explicit_self {
-            ty::StaticExplicitSelfCategory => { // reason (a) above
-                self.tcx().sess.span_err(
-                    self.span,
-                    "cannot call a method without a receiver \
-                     through an object");
-                return false;
-            }
-
-            ty::ByValueExplicitSelfCategory |
-            ty::ByReferenceExplicitSelfCategory(..) |
-            ty::ByBoxExplicitSelfCategory => {}
-        }
-
-        // reason (a) above
-        let check_for_self_ty = |ty| -> bool {
-            if ty::type_has_self(ty) {
-                span_err!(self.tcx().sess, self.span, E0038,
-                    "cannot call a method whose type contains a \
-                     self-type through an object");
-                false
-            } else {
-                true
-            }
-        };
-        let ref sig = candidate.method_ty.fty.sig;
-        for &input_ty in sig.inputs[1..].iter() {
-            if !check_for_self_ty(input_ty) {
-                return false;
-            }
-        }
-        if let ty::FnConverging(result_type) = sig.output {
-            if !check_for_self_ty(result_type) {
-                return false;
-            }
-        }
-
-        if candidate.method_ty.generics.has_type_params(subst::FnSpace) {
-            // reason (b) above
-            span_err!(self.tcx().sess, self.span, E0039,
-                "cannot call a generic method through an object");
-            return false;
-        }
-
-        true
-    }
-
     fn enforce_drop_trait_limitations(&self, candidate: &Candidate) {
         // No code can call the finalize method explicitly.
         let bad = match candidate.origin {

src/librustc/middle/typeck/check/mod.rs

@@ -1687,6 +1687,7 @@ fn register_unsize_obligations(&self,
                 self.register_unsize_obligations(span, &**u)
             }
             ty::UnsizeVtable(ref ty_trait, self_ty) => {
+                vtable2::check_object_safety(self.tcx(), ty_trait, span);
                 // If the type is `Foo+'a`, ensures that the type
                 // being cast to `Foo+'a` implements `Foo`:
                 vtable::register_object_cast_obligations(self,

src/librustc/middle/typeck/check/vtable.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use middle::subst::{SelfSpace};
+use middle::subst::{SelfSpace, FnSpace};
 use middle::traits;
 use middle::traits::{SelectionError, OutputTypeParameterMismatch, Overflow, Unimplemented};
 use middle::traits::{Obligation, obligation_for_builtin_bound};
@@ -46,6 +46,7 @@ pub fn check_object_cast(fcx: &FnCtxt,
 
             // Ensure that if ~T is cast to ~Trait, then T : Trait
             push_cast_obligation(fcx, cast_expr, object_trait, referent_ty);
+            check_object_safety(fcx.tcx(), object_trait, source_expr.span);
         }
 
         (&ty::ty_rptr(referent_region, ty::mt { ty: referent_ty,
@@ -68,6 +69,8 @@ pub fn check_object_cast(fcx: &FnCtxt,
                                infer::RelateObjectBound(source_expr.span),
                                target_region,
                                referent_region);
+
+                check_object_safety(fcx.tcx(), object_trait, source_expr.span);
             }
         }
 
@@ -128,6 +131,70 @@ fn push_cast_obligation(fcx: &FnCtxt,
     }
 }
 
+// TODO comment
+pub fn check_object_safety(tcx: &ty::ctxt, object_trait: &ty::TyTrait, span: Span) {
+    let trait_items = ty::trait_items(tcx, object_trait.def_id);
+    for item in trait_items.iter() {
+        match *item {
+            ty::MethodTraitItem(ref m) => check_object_safety_of_method(tcx, &**m, span),
+            ty::TypeTraitItem(_) => {}
+        }
+    }
+
+    // TODO error messages
+    fn check_object_safety_of_method(tcx: &ty::ctxt, method: &ty::Method, span: Span) {
+        /*!
+         * There are some limitations to calling functions through an
+         * object, because (a) the self type is not known
+         * (that's the whole point of a trait instance, after all, to
+         * obscure the self type) and (b) the call must go through a
+         * vtable and hence cannot be monomorphized.
+         */
+
+        match method.explicit_self {
+            ty::ByValueExplicitSelfCategory => { // reason (a) above
+                tcx.sess.span_err(
+                    span,
+                    "cannot call a method with a by-value receiver \
+                     through a trait object");
+            }
+
+            ty::StaticExplicitSelfCategory |
+            ty::ByReferenceExplicitSelfCategory(..) |
+            ty::ByBoxExplicitSelfCategory => {}
+        }
+
+        // reason (a) above
+        let check_for_self_ty = |ty| {
+            if ty::type_has_self(ty) {
+                span_err!(tcx.sess, span, E0038,
+                    "cannot call a method whose type contains a \
+                     self-type through an object: {}", ::util::ppaux::ty_to_string(tcx, ty));
+                true
+            } else {
+                false
+            }
+        };
+        let ref sig = method.fty.sig;
+        let mut found_self_ty = false;
+        for &input_ty in sig.inputs.tail().iter() {
+            if check_for_self_ty(input_ty) {
+                found_self_ty = true;
+                break;
+            }
+        }
+        if !found_self_ty {
+            check_for_self_ty(sig.output);
+        }
+
+        if method.generics.has_type_params(FnSpace) {
+            // reason (b) above
+            span_err!(tcx.sess, span, E0039,
+                "cannot call a generic method through an object");
+        }
+    }
+}
+
 pub fn register_object_cast_obligations(fcx: &FnCtxt,
                                         span: Span,
                                         object_trait: &ty::TyTrait,