Auto merge of #96837 - tmiasko:stdio-fcntl, r=joshtriplett

Use `fcntl(fd, F_GETFD)` to detect if standard streams are open

In the previous implementation, if the standard streams were open,
but the RLIMIT_NOFILE value was below three, the poll would fail
with EINVAL:

> ERRORS: EINVAL The nfds value exceeds the RLIMIT_NOFILE value.

Switch to the existing fcntl based implementation to avoid the issue.

Fixes #96621.

library/std/src/lib.rs

@@ -239,6 +239,7 @@
 #![feature(dropck_eyepatch)]
 #![feature(exhaustive_patterns)]
 #![feature(intra_doc_pointers)]
+#![feature(label_break_value)]
 #![feature(lang_items)]
 #![feature(let_chains)]
 #![feature(let_else)]

library/std/src/sys/unix/mod.rs

@@ -77,35 +77,65 @@ pub unsafe fn init(argc: isize, argv: *const *const u8) {
     }
 
     unsafe fn sanitize_standard_fds() {
-        #[cfg(not(miri))]
-        // The standard fds are always available in Miri.
-        cfg_if::cfg_if! {
-            if #[cfg(not(any(
-                target_os = "emscripten",
-                target_os = "fuchsia",
-                target_os = "vxworks",
-                // The poll on Darwin doesn't set POLLNVAL for closed fds.
-                target_os = "macos",
-                target_os = "ios",
-                target_os = "redox",
-                target_os = "l4re",
-            )))] {
-                use crate::sys::os::errno;
-                let pfds: &mut [_] = &mut [
-                    libc::pollfd { fd: 0, events: 0, revents: 0 },
-                    libc::pollfd { fd: 1, events: 0, revents: 0 },
-                    libc::pollfd { fd: 2, events: 0, revents: 0 },
-                ];
-                while libc::poll(pfds.as_mut_ptr(), 3, 0) == -1 {
-                    if errno() == libc::EINTR {
-                        continue;
+        // fast path with a single syscall for systems with poll()
+        #[cfg(not(any(
+            miri,
+            target_os = "emscripten",
+            target_os = "fuchsia",
+            target_os = "vxworks",
+            // The poll on Darwin doesn't set POLLNVAL for closed fds.
+            target_os = "macos",
+            target_os = "ios",
+            target_os = "redox",
+            target_os = "l4re",
+        )))]
+        'poll: {
+            use crate::sys::os::errno;
+            let pfds: &mut [_] = &mut [
+                libc::pollfd { fd: 0, events: 0, revents: 0 },
+                libc::pollfd { fd: 1, events: 0, revents: 0 },
+                libc::pollfd { fd: 2, events: 0, revents: 0 },
+            ];
+
+            while libc::poll(pfds.as_mut_ptr(), 3, 0) == -1 {
+                match errno() {
+                    libc::EINTR => continue,
+                    libc::EINVAL | libc::EAGAIN | libc::ENOMEM => {
+                        // RLIMIT_NOFILE or temporary allocation failures
+                        // may be preventing use of poll(), fall back to fcntl
+                        break 'poll;
                     }
+                    _ => libc::abort(),
+                }
+            }
+            for pfd in pfds {
+                if pfd.revents & libc::POLLNVAL == 0 {
+                    continue;
+                }
+                if libc::open("/dev/null\0".as_ptr().cast(), libc::O_RDWR, 0) == -1 {
+                    // If the stream is closed but we failed to reopen it, abort the
+                    // process. Otherwise we wouldn't preserve the safety of
+                    // operations on the corresponding Rust object Stdin, Stdout, or
+                    // Stderr.
                     libc::abort();
                 }
-                for pfd in pfds {
-                    if pfd.revents & libc::POLLNVAL == 0 {
-                        continue;
-                    }
+            }
+            return;
+        }
+
+        // fallback in case poll isn't available or limited by RLIMIT_NOFILE
+        #[cfg(not(any(
+            // The standard fds are always available in Miri.
+            miri,
+            target_os = "emscripten",
+            target_os = "fuchsia",
+            target_os = "vxworks",
+            target_os = "l4re",
+        )))]
+        {
+            use crate::sys::os::errno;
+            for fd in 0..3 {
+                if libc::fcntl(fd, libc::F_GETFD) == -1 && errno() == libc::EBADF {
                     if libc::open("/dev/null\0".as_ptr().cast(), libc::O_RDWR, 0) == -1 {
                         // If the stream is closed but we failed to reopen it, abort the
                         // process. Otherwise we wouldn't preserve the safety of
@@ -114,15 +144,6 @@ unsafe fn sanitize_standard_fds() {
                         libc::abort();
                     }
                 }
-            } else if #[cfg(any(target_os = "macos", target_os = "ios", target_os = "redox"))] {
-                use crate::sys::os::errno;
-                for fd in 0..3 {
-                    if libc::fcntl(fd, libc::F_GETFD) == -1 && errno() == libc::EBADF {
-                        if libc::open("/dev/null\0".as_ptr().cast(), libc::O_RDWR, 0) == -1 {
-                            libc::abort();
-                        }
-                    }
-                }
             }
         }
     }

src/test/ui/process/nofile-limit.rs

+// Check that statically linked binary executes successfully
+// with RLIMIT_NOFILE resource lowered to zero. Regression
+// test for issue #96621.
+//
+// run-pass
+// dont-check-compiler-stderr
+// only-linux
+// no-prefer-dynamic
+// compile-flags: -Ctarget-feature=+crt-static -Crpath=no
+#![feature(exit_status_error)]
+#![feature(rustc_private)]
+extern crate libc;
+
+use std::os::unix::process::CommandExt;
+use std::process::Command;
+
+fn main() {
+    let mut args = std::env::args();
+    let this = args.next().unwrap();
+    match args.next().as_deref() {
+        None => {
+            let mut cmd = Command::new(this);
+            cmd.arg("Ok!");
+            unsafe {
+                cmd.pre_exec(|| {
+                    let rlim = libc::rlimit {
+                        rlim_cur: 0,
+                        rlim_max: 0,
+                    };
+                    if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) == -1 {
+                        Err(std::io::Error::last_os_error())
+                    } else {
+                        Ok(())
+                    }
+                })
+            };
+            let output = cmd.output().unwrap();
+            println!("{:?}", output);
+            output.status.exit_ok().unwrap();
+            assert!(output.stdout.starts_with(b"Ok!"));
+        }
+        Some(word) => {
+            println!("{}", word);
+        }
+    }
+}