Auto merge of #38072 - nikomatsakis:bootstrap-incremental, r=acrichto

add preliminary support for incremental compilation to rustbuild.py

This implements the integration described in #37929. It requires the use of a local nightly as your bootstrap compiler. The setup is described in `src/bootstrap/README.md`.

This does NOT implement the "copy stage0 libs to stage1" optimization described in #37929, just because that seems orthogonal to me.

In local testing, I do not yet see any incremental re-use when building rustc. I'm not sure why that is, more investigation needed.

(For these reasons, this is not marked as fixing the relevant issue.)

r? @alexcrichton -- I included one random cleanup (`Step::noop()`) that turned out to not be especially relevant. Feel free to tell me you liked it better the old way.

src/bootstrap/README.md

@@ -116,6 +116,42 @@ compiler. What actually happens when you invoke rustbuild is:
 The goal of each stage is to (a) leverage Cargo as much as possible and failing
 that (b) leverage Rust as much as possible!
 
+## Incremental builds
+
+You can configure rustbuild to use incremental compilation. Because
+incremental is new and evolving rapidly, if you want to use it, it is
+recommended that you replace the snapshot with a locally installed
+nightly build of rustc. You will want to keep this up to date.
+
+To follow this course of action, first thing you will want to do is to
+install a nightly, presumably using `rustup`. You will then want to
+configure your directory to use this build, like so:
+
+```
+# configure to use local rust instead of downloding a beta.
+# `--local-rust-root` is optional here. If elided, we will
+# use whatever rustc we find on your PATH.
+> configure --enable-rustbuild --local-rust-root=~/.cargo/ --enable-local-rebuild
+```
+
+After that, you can use the `--incremental` flag to actually do
+incremental builds:
+
+```
+> ../x.py build --incremental
+```
+
+The `--incremental` flag will store incremental compilation artifacts
+in `build/stage0-incremental`. Note that we only use incremental
+compilation for the stage0 -> stage1 compilation -- this is because
+the stage1 compiler is changing, and we don't try to cache and reuse
+incremental artifacts across different versions of the compiler. For
+this reason, `--incremental` defaults to `--stage 1` (though you can
+manually select a higher stage, if you prefer).
+
+You can always drop the `--incremental` to build as normal (but you
+will still be using the local nightly as your bootstrap).
+
 ## Directory Layout
 
 This build system houses all output under the `build` directory, which looks

src/bootstrap/bin/rustc.rs

@@ -29,6 +29,9 @@
 
 use std::env;
 use std::ffi::OsString;
+use std::io;
+use std::io::prelude::*;
+use std::str::FromStr;
 use std::path::PathBuf;
 use std::process::{Command, ExitStatus};
 
@@ -41,6 +44,11 @@ fn main() {
         .and_then(|w| w[1].to_str());
     let version = args.iter().find(|w| &**w == "-vV");
 
+    let verbose = match env::var("RUSTC_VERBOSE") {
+        Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"),
+        Err(_) => 0,
+    };
+
     // Build scripts always use the snapshot compiler which is guaranteed to be
     // able to produce an executable, whereas intermediate compilers may not
     // have the standard library built yet and may not be able to produce an
@@ -95,6 +103,15 @@ fn main() {
             cmd.args(&s.split(" ").filter(|s| !s.is_empty()).collect::<Vec<_>>());
         }
 
+        // Pass down incremental directory, if any.
+        if let Ok(dir) = env::var("RUSTC_INCREMENTAL") {
+            cmd.arg(format!("-Zincremental={}", dir));
+
+            if verbose > 0 {
+                cmd.arg("-Zincremental-info");
+            }
+        }
+
         // If we're compiling specifically the `panic_abort` crate then we pass
         // the `-C panic=abort` option. Note that we do not do this for any
         // other crate intentionally as this is the only crate for now that we
@@ -176,9 +193,19 @@ fn main() {
             if let Some(rpath) = rpath {
                 cmd.arg("-C").arg(format!("link-args={}", rpath));
             }
+
+            if let Ok(s) = env::var("RUSTFLAGS") {
+                for flag in s.split_whitespace() {
+                    cmd.arg(flag);
+                }
+            }
         }
     }
 
+    if verbose > 1 {
+        writeln!(&mut io::stderr(), "rustc command: {:?}", cmd).unwrap();
+    }
+
     // Actually run the compiler!
     std::process::exit(match exec_cmd(&mut cmd) {
         Ok(s) => s.code().unwrap_or(0xfe),

src/bootstrap/bootstrap.py

@@ -294,6 +294,8 @@ class RustBuild(object):
         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                       os.pathsep + env["PATH"]
+        if not os.path.isfile(self.cargo()):
+            raise Exception("no cargo executable found at `%s`" % self.cargo())
         args = [self.cargo(), "build", "--manifest-path",
                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
         if self.use_vendored_sources:

src/bootstrap/check.rs

@@ -190,7 +190,7 @@ pub fn compiletest(build: &Build,
 
     cmd.args(&build.flags.cmd.test_args());
 
-    if build.config.verbose || build.flags.verbose {
+    if build.config.verbose() || build.flags.verbose() {
         cmd.arg("--verbose");
     }
 

src/bootstrap/config.rs

@@ -40,7 +40,7 @@
 pub struct Config {
     pub ccache: Option<String>,
     pub ninja: bool,
-    pub verbose: bool,
+    pub verbose: usize,
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
@@ -504,6 +504,14 @@ pub fn update_with_config_mk(&mut self) {
             }
         }
     }
+
+    pub fn verbose(&self) -> bool {
+        self.verbose > 0
+    }
+
+    pub fn very_verbose(&self) -> bool {
+        self.verbose > 1
+    }
 }
 
 #[cfg(not(windows))]

src/bootstrap/flags.rs

@@ -27,7 +27,7 @@
 
 /// Deserialized version of all flags for this compile.
 pub struct Flags {
-    pub verbose: bool,
+    pub verbose: usize, // verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
     pub stage: Option<u32>,
     pub keep_stage: Option<u32>,
     pub build: String,
@@ -37,6 +37,17 @@ pub struct Flags {
     pub src: Option<PathBuf>,
     pub jobs: Option<u32>,
     pub cmd: Subcommand,
+    pub incremental: bool,
+}
+
+impl Flags {
+    pub fn verbose(&self) -> bool {
+        self.verbose > 0
+    }
+
+    pub fn very_verbose(&self) -> bool {
+        self.verbose > 1
+    }
 }
 
 pub enum Subcommand {
@@ -63,7 +74,8 @@ pub enum Subcommand {
 impl Flags {
     pub fn parse(args: &[String]) -> Flags {
         let mut opts = Options::new();
-        opts.optflag("v", "verbose", "use verbose output");
+        opts.optflagmulti("v", "verbose", "use verbose output (-vv for very verbose)");
+        opts.optflag("i", "incremental", "use incremental compilation");
         opts.optopt("", "config", "TOML configuration file for build", "FILE");
         opts.optopt("", "build", "build target of the stage0 compiler", "BUILD");
         opts.optmulti("", "host", "host targets to build", "HOST");
@@ -256,8 +268,18 @@ pub fn parse(args: &[String]) -> Flags {
             }
         });
 
+        let mut stage = m.opt_str("stage").map(|j| j.parse().unwrap());
+
+        let incremental = m.opt_present("i");
+
+        if incremental {
+            if stage.is_none() {
+                stage = Some(1);
+            }
+        }
+
         Flags {
-            verbose: m.opt_present("v"),
+            verbose: m.opt_count("v"),
             stage: m.opt_str("stage").map(|j| j.parse().unwrap()),
             keep_stage: m.opt_str("keep-stage").map(|j| j.parse().unwrap()),
             build: m.opt_str("build").unwrap_or_else(|| {
@@ -269,6 +291,7 @@ pub fn parse(args: &[String]) -> Flags {
             src: m.opt_str("src").map(PathBuf::from),
             jobs: m.opt_str("jobs").map(|j| j.parse().unwrap()),
             cmd: cmd,
+            incremental: incremental,
         }
     }
 }

src/bootstrap/lib.rs

@@ -74,6 +74,7 @@
 extern crate toml;
 
 use std::collections::HashMap;
+use std::cmp;
 use std::env;
 use std::ffi::OsString;
 use std::fs::{self, File};
@@ -497,6 +498,17 @@ fn cargo(&self,
         cargo.env("RUSTC_BOOTSTRAP", "1");
         self.add_rust_test_threads(&mut cargo);
 
+        // Ignore incremental modes except for stage0, since we're
+        // not guaranteeing correctness acros builds if the compiler
+        // is changing under your feet.`
+        if self.flags.incremental && compiler.stage == 0 {
+            let incr_dir = self.incremental_dir(compiler);
+            cargo.env("RUSTC_INCREMENTAL", incr_dir);
+        }
+
+        let verbose = cmp::max(self.config.verbose, self.flags.verbose);
+        cargo.env("RUSTC_VERBOSE", format!("{}", verbose));
+
         // Specify some various options for build scripts used throughout
         // the build.
         //
@@ -516,7 +528,7 @@ fn cargo(&self,
         // FIXME: should update code to not require this env var
         cargo.env("CFG_COMPILER_HOST_TRIPLE", target);
 
-        if self.config.verbose || self.flags.verbose {
+        if self.config.verbose() || self.flags.verbose() {
             cargo.arg("-v");
         }
         // FIXME: cargo bench does not accept `--release`
@@ -630,6 +642,12 @@ fn sysroot(&self, compiler: &Compiler) -> PathBuf {
         }
     }
 
+    /// Get the directory for incremental by-products when using the
+    /// given compiler.
+    fn incremental_dir(&self, compiler: &Compiler) -> PathBuf {
+        self.out.join(compiler.host).join(format!("stage{}-incremental", compiler.stage))
+    }
+
     /// Returns the libdir where the standard library and other artifacts are
     /// found for a compiler's sysroot.
     fn sysroot_libdir(&self, compiler: &Compiler, target: &str) -> PathBuf {
@@ -768,7 +786,7 @@ fn run(&self, cmd: &mut Command) {
 
     /// Prints a message if this build is configured in verbose mode.
     fn verbose(&self, msg: &str) {
-        if self.flags.verbose || self.config.verbose {
+        if self.flags.verbose() || self.config.verbose() {
             println!("{}", msg);
         }
     }

src/bootstrap/step.rs

@@ -86,7 +86,7 @@ pub fn build_rules(build: &Build) -> Rules {
     //
     // To handle this we do a bit of dynamic dispatch to see what the dependency
     // is. If we're building a LLVM for the build triple, then we don't actually
-    // have any dependencies! To do that we return a dependency on the "dummy"
+    // have any dependencies! To do that we return a dependency on the `Step::noop()`
     // target which does nothing.
     //
     // If we're build a cross-compiled LLVM, however, we need to assemble the
@@ -104,7 +104,7 @@ pub fn build_rules(build: &Build) -> Rules {
          .host(true)
          .dep(move |s| {
              if s.target == build.config.build {
-                 dummy(s, build)
+                 Step::noop()
              } else {
                  s.target(&build.config.build)
              }
@@ -115,14 +115,11 @@ pub fn build_rules(build: &Build) -> Rules {
     // going on here. You can check out the API docs below and also see a bunch
     // more examples of rules directly below as well.
 
-    // dummy rule to do nothing, useful when a dep maps to no deps
-    rules.build("dummy", "path/to/nowhere");
-
     // the compiler with no target libraries ready to go
     rules.build("rustc", "src/rustc")
          .dep(move |s| {
              if s.stage == 0 {
-                 dummy(s, build)
+                 Step::noop()
              } else {
                  s.name("librustc")
                   .host(&build.config.build)
@@ -165,7 +162,7 @@ pub fn build_rules(build: &Build) -> Rules {
              .dep(move |s| s.name("rustc").host(&build.config.build).target(s.host))
              .dep(move |s| {
                  if s.host == build.config.build {
-                    dummy(s, build)
+                     Step::noop()
                  } else {
                     s.host(&build.config.build)
                  }
@@ -183,7 +180,7 @@ pub fn build_rules(build: &Build) -> Rules {
              .dep(|s| s.name("libstd"))
              .dep(move |s| {
                  if s.host == build.config.build {
-                    dummy(s, build)
+                    Step::noop()
                  } else {
                     s.host(&build.config.build)
                  }
@@ -203,7 +200,7 @@ pub fn build_rules(build: &Build) -> Rules {
              .dep(move |s| s.name("llvm").host(&build.config.build).stage(0))
              .dep(move |s| {
                  if s.host == build.config.build {
-                    dummy(s, build)
+                    Step::noop()
                  } else {
                     s.host(&build.config.build)
                  }
@@ -233,7 +230,7 @@ pub fn build_rules(build: &Build) -> Rules {
                      if s.target.contains("android") {
                          s.name("android-copy-libs")
                      } else {
-                         dummy(s, build)
+                         Step::noop()
                      }
                  })
                  .default(true)
@@ -514,12 +511,6 @@ pub fn build_rules(build: &Build) -> Rules {
 
     rules.verify();
     return rules;
-
-    fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> {
-        s.name("dummy").stage(0)
-         .target(&build.config.build)
-         .host(&build.config.build)
-    }
 }
 
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
@@ -543,6 +534,10 @@ struct Step<'a> {
 }
 
 impl<'a> Step<'a> {
+    fn noop() -> Step<'a> {
+        Step { name: "", stage: 0, host: "", target: "" }
+    }
+
     /// Creates a new step which is the same as this, except has a new name.
     fn name(&self, name: &'a str) -> Step<'a> {
         Step { name: name, ..*self }
@@ -738,6 +733,9 @@ fn verify(&self) {
                 if self.rules.contains_key(&dep.name) || dep.name.starts_with("default:") {
                     continue
                 }
+                if dep == Step::noop() {
+                    continue
+                }
                 panic!("\
 
 invalid rule dependency graph detected, was a rule added and maybe typo'd?
@@ -864,6 +862,7 @@ fn run(&self, steps: &[Step<'a>]) {
         // of what we need to do.
         let mut order = Vec::new();
         let mut added = HashSet::new();
+        added.insert(Step::noop());
         for step in steps.iter().cloned() {
             self.fill(step, &mut order, &mut added);
         }