Auto merge of #67362 - Mark-Simulacrum:par-4-default, r=alexcrichton

4 thread parallelism by default

The Session default here is super unusual but seems to both compile and do what we expect as best as I can tell.

src/bootstrap/test.rs

@@ -4,10 +4,10 @@
 //! our CI.
 
 use std::env;
-use std::ffi::OsString;
+//use std::ffi::OsString;
 use std::fmt;
 use std::fs;
-use std::iter;
+//use std::iter;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
@@ -204,8 +204,8 @@ fn make_run(run: RunConfig<'_>) {
     }
 
     /// Runs `cargo test` for `cargo` packaged with Rust.
-    fn run(self, builder: &Builder<'_>) {
-        let compiler = builder.compiler(self.stage, self.host);
+    fn run(self, _builder: &Builder<'_>) {
+        /*let compiler = builder.compiler(self.stage, self.host);
 
         builder.ensure(tool::Cargo {
             compiler,
@@ -235,7 +235,7 @@ fn run(self, builder: &Builder<'_>) {
 
         cargo.env("PATH", &path_for_cargo(builder, compiler));
 
-        try_run(builder, &mut cargo.into());
+        try_run(builder, &mut cargo.into());*/
     }
 }
 
@@ -590,14 +590,14 @@ fn run(self, builder: &Builder<'_>) {
     }
 }
 
-fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString {
-    // Configure PATH to find the right rustc. NB. we have to use PATH
-    // and not RUSTC because the Cargo test suite has tests that will
-    // fail if rustc is not spelled `rustc`.
-    let path = builder.sysroot(compiler).join("bin");
-    let old_path = env::var_os("PATH").unwrap_or_default();
-    env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("")
-}
+//fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString {
+//    // Configure PATH to find the right rustc. NB. we have to use PATH
+//    // and not RUSTC because the Cargo test suite has tests that will
+//    // fail if rustc is not spelled `rustc`.
+//    let path = builder.sysroot(compiler).join("bin");
+//    let old_path = env::var_os("PATH").unwrap_or_default();
+//    env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("")
+//}
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct RustdocTheme {

src/ci/run.sh

@@ -37,6 +37,8 @@ if [ "$DIST_SRC" = "" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"
 fi
 
+RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler"
+
 # If we're deploying artifacts then we set the release channel, otherwise if
 # we're not deploying then we want to be sure to enable all assertions because
 # we'll be running tests
@@ -53,9 +55,6 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
   if [ "$NO_LLVM_ASSERTIONS" = "1" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-llvm-assertions"
   elif [ "$DEPLOY_ALT" != "" ]; then
-    if [ "$NO_PARALLEL_COMPILER" = "" ]; then
-      RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler"
-    fi
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
   fi

src/librustc/dep_graph/graph.rs

@@ -3,7 +3,7 @@
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_index::vec::{Idx, IndexVec};
 use smallvec::SmallVec;
-use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicU64, Ordering};
+use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicUsize, Ordering};
 use rustc_data_structures::sharded::{self, Sharded};
 use std::sync::atomic::Ordering::SeqCst;
 use std::env;
@@ -485,8 +485,8 @@ pub fn edge_deduplication_data(&self) -> Option<(u64, u64)> {
         if cfg!(debug_assertions) {
             let current_dep_graph = &self.data.as_ref().unwrap().current;
 
-            Some((current_dep_graph.total_read_count.load(SeqCst),
-                  current_dep_graph.total_duplicate_read_count.load(SeqCst)))
+            Some((current_dep_graph.total_read_count.load(SeqCst) as u64,
+                  current_dep_graph.total_duplicate_read_count.load(SeqCst) as u64))
         } else {
             None
         }
@@ -970,8 +970,8 @@ pub(super) struct CurrentDepGraph {
 
     /// These are simple counters that are for profiling and
     /// debugging and only active with `debug_assertions`.
-    total_read_count: AtomicU64,
-    total_duplicate_read_count: AtomicU64,
+    total_read_count: AtomicUsize,
+    total_duplicate_read_count: AtomicUsize,
 }
 
 impl CurrentDepGraph {
@@ -1012,8 +1012,8 @@ fn new(prev_graph_node_count: usize) -> CurrentDepGraph {
             )),
             anon_id_seed: stable_hasher.finish(),
             forbidden_edge,
-            total_read_count: AtomicU64::new(0),
-            total_duplicate_read_count: AtomicU64::new(0),
+            total_read_count: AtomicUsize::new(0),
+            total_duplicate_read_count: AtomicUsize::new(0),
         }
     }
 

src/librustc_data_structures/lib.rs

@@ -24,6 +24,7 @@
 #![feature(integer_atomics)]
 #![feature(test)]
 #![feature(associated_type_bounds)]
+#![feature(cfg_target_has_atomic)]
 
 #![cfg_attr(unix, feature(libc))]
 

src/librustc_data_structures/sync.rs

@@ -317,7 +317,9 @@ fn clone(&self) -> Self {
         pub use parking_lot::MutexGuard as LockGuard;
         pub use parking_lot::MappedMutexGuard as MappedLockGuard;
 
-        pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64};
+        pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32};
+        #[cfg(target_has_atomic = "64")]
+        pub use std::sync::atomic::{AtomicU64};
 
         pub use crossbeam_utils::atomic::AtomicCell;
 

src/librustc_session/config.rs

@@ -1358,11 +1358,11 @@ fn parse_symbol_mangling_version(
         "prints the LLVM optimization passes being run"),
     ast_json: bool = (false, parse_bool, [UNTRACKED],
         "print the AST as JSON and halt"),
-    // We default to 1 here since we want to behave like
-    // a sequential compiler for now. This'll likely be adjusted
-    // in the future. Note that -Zthreads=0 is the way to get
-    // the num_cpus behavior.
-    threads: usize = (1, parse_threads, [UNTRACKED],
+    // We default to min(4, vCPUs) here since we want to avoid spawning *too*
+    // many threads -- that causes scalability issues due to contention on
+    // the jobserver pipe (at least) -- but 4 is a reasonable amount on systems
+    // with lots of cores.
+    threads: usize = (std::cmp::min(::num_cpus::get(), 4), parse_threads, [UNTRACKED],
         "use a thread pool with N threads"),
     ast_json_noexpand: bool = (false, parse_bool, [UNTRACKED],
         "print the pre-expansion AST as JSON and halt"),

src/librustc_session/session.rs

@@ -14,7 +14,7 @@
 
 use rustc_data_structures::base_n;
 use rustc_data_structures::sync::{
-    self, Lrc, Lock, OneThread, Once, AtomicU64, AtomicUsize, Ordering,
+    self, Lrc, Lock, OneThread, Once, AtomicUsize, Ordering,
     Ordering::SeqCst,
 };
 use rustc_data_structures::impl_stable_hash_via_hash;
@@ -119,7 +119,7 @@ pub struct Session {
     /// If `-zprint-fuel=crate`, `Some(crate)`.
     pub print_fuel_crate: Option<String>,
     /// Always set to zero and incremented so that we can print fuel expended by a crate.
-    pub print_fuel: AtomicU64,
+    pub print_fuel: AtomicUsize,
 
     /// Loaded up early on in the initialization of this `Session` to avoid
     /// false positives about a job server in our environment.
@@ -1116,7 +1116,7 @@ fn build_session_(
         out_of_fuel: false,
     });
     let print_fuel_crate = sopts.debugging_opts.print_fuel.clone();
-    let print_fuel = AtomicU64::new(0);
+    let print_fuel = AtomicUsize::new(0);
 
     let working_dir = env::current_dir().unwrap_or_else(|e|
         parse_sess.span_diagnostic

src/test/ui/traits/cycle-cache-err-60010.rs

@@ -28,7 +28,7 @@ struct SalsaStorage {
 }
 
 impl Database for RootDatabase {
-    type Storage = SalsaStorage; //~ ERROR overflow
+    type Storage = SalsaStorage;
 }
 impl HasQueryGroup for RootDatabase {}
 impl<DB> Query<DB> for ParseQuery

src/test/ui/traits/cycle-cache-err-60010.stderr

@@ -6,22 +6,6 @@ LL |     _parse: <ParseQuery as Query<RootDatabase>>::Data,
    |
    = note: required because of the requirements on the impl of `Query<RootDatabase>` for `ParseQuery`
 
-error[E0275]: overflow evaluating the requirement `Runtime<RootDatabase>: std::panic::RefUnwindSafe`
-  --> $DIR/cycle-cache-err-60010.rs:31:5
-   |
-LL |     type Storage;
-   |          ------- associated type defined here
-...
-LL | impl Database for RootDatabase {
-   | ------------------------------ in this `impl` item
-LL |     type Storage = SalsaStorage;
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: required because it appears within the type `RootDatabase`
-   = note: required because of the requirements on the impl of `SourceDatabase` for `RootDatabase`
-   = note: required because of the requirements on the impl of `Query<RootDatabase>` for `ParseQuery`
-   = note: required because it appears within the type `SalsaStorage`
-
-error: aborting due to 2 previous errors
+error: aborting due to previous error
 
 For more information about this error, try `rustc --explain E0275`.