rustbuild: Implement `make dist`

This commit implements the `make dist` command in the new rustbuild build
system, porting over `dist.mk` and `prepare.mk` into Rust. There's a huge amount
of complexity between those two files, not all of which is likely justified, so
the Rust implementation is *much* smaller.

Currently the implementation still shells out to rust-installer as well as some
python scripts, but ideally we'd rewrite it all in the future to not shell out
and be in Rust proper.

src/bootstrap/build/channel.rs

@@ -36,19 +36,23 @@ pub fn collect(build: &mut Build) {
     match &build.config.channel[..] {
         "stable" => {
             build.release = release_num.to_string();
+            build.package_vers = build.release.clone();
             build.unstable_features = false;
         }
         "beta" => {
             build.release = format!("{}-beta{}", release_num,
                                    prerelease_version);
+            build.package_vers = "beta".to_string();
             build.unstable_features = false;
         }
         "nightly" => {
             build.release = format!("{}-nightly", release_num);
+            build.package_vers = "nightly".to_string();
             build.unstable_features = true;
         }
         _ => {
             build.release = format!("{}-dev", release_num);
+            build.package_vers = build.release.clone();
             build.unstable_features = true;
         }
     }

src/bootstrap/build/dist.rs

+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::fs::{self, File};
+use std::io::Write;
+use std::path::{PathBuf, Path};
+use std::process::Command;
+
+use build::{Build, Compiler};
+use build::util::{cp_r, libdir, is_dylib};
+
+fn package_vers(build: &Build) -> &str {
+    match &build.config.channel[..] {
+        "stable" => &build.release,
+        "beta" => "beta",
+        "nightly" => "nightly",
+        _ => &build.release,
+    }
+}
+
+fn distdir(build: &Build) -> PathBuf {
+    build.out.join("dist")
+}
+
+fn tmpdir(build: &Build) -> PathBuf {
+    build.out.join("tmp/dist")
+}
+
+pub fn docs(build: &Build, stage: u32, host: &str) {
+    println!("Dist docs stage{} ({})", stage, host);
+    let name = format!("rust-docs-{}", package_vers(build));
+    let image = tmpdir(build).join(format!("{}-{}-image", name, name));
+    let _ = fs::remove_dir_all(&image);
+
+    let dst = image.join("share/doc/rust/html");
+    t!(fs::create_dir_all(&dst));
+    let src = build.out.join(host).join("doc");
+    cp_r(&src, &dst);
+
+    let mut cmd = Command::new("sh");
+    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+       .arg("--product-name=Rust-Documentation")
+       .arg("--rel-manifest-dir=rustlib")
+       .arg("--success-message=Rust-documentation-is-installed.")
+       .arg(format!("--image-dir={}", sanitize_sh(&image)))
+       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
+       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg(format!("--package-name={}", name))
+       .arg("--component-name=rust-docs")
+       .arg("--legacy-manifest-dirs=rustlib,cargo")
+       .arg("--bulk-dirs=share/doc/rust/html");
+    build.run(&mut cmd);
+    t!(fs::remove_dir_all(&image));
+
+    // As part of this step, *also* copy the docs directory to a directory which
+    // buildbot typically uploads.
+    let dst = distdir(build).join("doc").join(&build.package_vers);
+    t!(fs::create_dir_all(&dst));
+    cp_r(&src, &dst);
+}
+
+pub fn mingw(build: &Build, host: &str) {
+    println!("Dist mingw ({})", host);
+    let name = format!("rust-mingw-{}", package_vers(build));
+    let image = tmpdir(build).join(format!("{}-{}-image", name, host));
+    let _ = fs::remove_dir_all(&image);
+
+    // The first argument to the script is a "temporary directory" which is just
+    // thrown away (this contains the runtime DLLs included in the rustc package
+    // above) and the second argument is where to place all the MinGW components
+    // (which is what we want).
+    //
+    // FIXME: this script should be rewritten into Rust
+    let mut cmd = Command::new("python");
+    cmd.arg(build.src.join("src/etc/make-win-dist.py"))
+       .arg(tmpdir(build))
+       .arg(&image)
+       .arg(host);
+    build.run(&mut cmd);
+
+    let mut cmd = Command::new("sh");
+    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+       .arg("--product-name=Rust-MinGW")
+       .arg("--rel-manifest-dir=rustlib")
+       .arg("--success-message=Rust-MinGW-is-installed.")
+       .arg(format!("--image-dir={}", sanitize_sh(&image)))
+       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
+       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg(format!("--package-name={}-{}", name, host))
+       .arg("--component-name=rust-mingw")
+       .arg("--legacy-manifest-dirs=rustlib,cargo");
+    build.run(&mut cmd);
+    t!(fs::remove_dir_all(&image));
+}
+
+pub fn rustc(build: &Build, stage: u32, host: &str) {
+    println!("Dist rustc stage{} ({})", stage, host);
+    let name = format!("rustc-{}", package_vers(build));
+    let image = tmpdir(build).join(format!("{}-{}-image", name, host));
+    let _ = fs::remove_dir_all(&image);
+    let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
+    let _ = fs::remove_dir_all(&overlay);
+
+    // Prepare the rustc "image", what will actually end up getting installed
+    prepare_image(build, stage, host, &image);
+
+    // Prepare the overlay which is part of the tarball but won't actually be
+    // installed
+    t!(fs::create_dir_all(&overlay));
+    let cp = |file: &str| {
+        install(&build.src.join(file), &overlay, 0o644);
+    };
+    cp("COPYRIGHT");
+    cp("LICENSE-APACHE");
+    cp("LICENSE-MIT");
+    cp("README.md");
+    // tiny morsel of metadata is used by rust-packaging
+    let version = &build.version;
+    t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
+
+    // On MinGW we've got a few runtime DLL dependencies that we need to
+    // include. The first argument to this script is where to put these DLLs
+    // (the image we're creating), and the second argument is a junk directory
+    // to ignore all other MinGW stuff the script creates.
+    //
+    // On 32-bit MinGW we're always including a DLL which needs some extra
+    // licenses to distribute. On 64-bit MinGW we don't actually distribute
+    // anything requiring us to distribute a license, but it's likely the
+    // install will *also* include the rust-mingw package, which also needs
+    // licenses, so to be safe we just include it here in all MinGW packages.
+    //
+    // FIXME: this script should be rewritten into Rust
+    if host.contains("pc-windows-gnu") {
+        let mut cmd = Command::new("python");
+        cmd.arg(build.src.join("src/etc/make-win-dist.py"))
+           .arg(&image)
+           .arg(tmpdir(build))
+           .arg(host);
+        build.run(&mut cmd);
+
+        let dst = image.join("share/doc");
+        t!(fs::create_dir_all(&dst));
+        cp_r(&build.src.join("src/etc/third-party"), &dst);
+    }
+
+    // Finally, wrap everything up in a nice tarball!
+    let mut cmd = Command::new("sh");
+    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+       .arg("--product-name=Rust")
+       .arg("--rel-manifest-dir=rustlib")
+       .arg("--success-message=Rust-is-ready-to-roll.")
+       .arg(format!("--image-dir={}", sanitize_sh(&image)))
+       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
+       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay)))
+       .arg(format!("--package-name={}-{}", name, host))
+       .arg("--component-name=rustc")
+       .arg("--legacy-manifest-dirs=rustlib,cargo");
+    build.run(&mut cmd);
+    t!(fs::remove_dir_all(&image));
+    t!(fs::remove_dir_all(&overlay));
+
+    fn prepare_image(build: &Build, stage: u32, host: &str, image: &Path) {
+        let src = build.sysroot(&Compiler::new(stage, host));
+        let libdir = libdir(host);
+
+        // Copy rustc/rustdoc binaries
+        t!(fs::create_dir_all(image.join("bin")));
+        cp_r(&src.join("bin"), &image.join("bin"));
+
+        // Copy runtime DLLs needed by the compiler
+        if libdir != "bin" {
+            t!(fs::create_dir_all(image.join(libdir)));
+            for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
+                let name = entry.file_name();
+                if let Some(s) = name.to_str() {
+                    if is_dylib(s) {
+                        install(&entry.path(), &image.join(libdir), 0o644);
+                    }
+                }
+            }
+        }
+
+        // Man pages
+        t!(fs::create_dir_all(image.join("share/man/man1")));
+        cp_r(&build.src.join("man"), &image.join("share/man/man1"));
+
+        // Debugger scripts
+        let cp_debugger_script = |file: &str| {
+            let dst = image.join("lib/rustlib/etc");
+            t!(fs::create_dir_all(&dst));
+            install(&build.src.join("src/etc/").join(file), &dst, 0o644);
+        };
+        if host.contains("windows") {
+            // no debugger scripts
+        } else if host.contains("darwin") {
+            // lldb debugger scripts
+            install(&build.src.join("src/etc/rust-lldb"), &image.join("bin"),
+                    0o755);
+
+            cp_debugger_script("lldb_rust_formatters.py");
+            cp_debugger_script("debugger_pretty_printers_common.py");
+        } else {
+            // gdb debugger scripts
+            install(&build.src.join("src/etc/rust-gdb"), &image.join("bin"),
+                    0o755);
+
+            cp_debugger_script("gdb_load_rust_pretty_printers.py");
+            cp_debugger_script("gdb_rust_pretty_printing.py");
+            cp_debugger_script("debugger_pretty_printers_common.py");
+        }
+
+        // Misc license info
+        let cp = |file: &str| {
+            install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
+        };
+        t!(fs::create_dir_all(&image.join("share/doc/rust")));
+        cp("COPYRIGHT");
+        cp("LICENSE-APACHE");
+        cp("LICENSE-MIT");
+        cp("README.md");
+    }
+}
+
+pub fn std(build: &Build, compiler: &Compiler, target: &str) {
+    println!("Dist std stage{} ({} -> {})", compiler.stage, compiler.host,
+             target);
+    let name = format!("rust-std-{}", package_vers(build));
+    let image = tmpdir(build).join(format!("{}-{}-image", name, target));
+    let _ = fs::remove_dir_all(&image);
+
+    let dst = image.join("lib/rustlib").join(target);
+    t!(fs::create_dir_all(&dst));
+    let src = build.sysroot(compiler).join("lib/rustlib");
+    cp_r(&src.join(target), &dst);
+
+    let mut cmd = Command::new("sh");
+    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+       .arg("--product-name=Rust")
+       .arg("--rel-manifest-dir=rustlib")
+       .arg("--success-message=std-is-standing-at-the-ready.")
+       .arg(format!("--image-dir={}", sanitize_sh(&image)))
+       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
+       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg(format!("--package-name={}-{}", name, target))
+       .arg(format!("--component-name=rust-std-{}", target))
+       .arg("--legacy-manifest-dirs=rustlib,cargo");
+    build.run(&mut cmd);
+    t!(fs::remove_dir_all(&image));
+}
+
+fn install(src: &Path, dstdir: &Path, perms: u32) {
+    let dst = dstdir.join(src.file_name().unwrap());
+    t!(fs::copy(src, &dst));
+    chmod(&dst, perms);
+}
+
+#[cfg(unix)]
+fn chmod(path: &Path, perms: u32) {
+    use std::os::unix::fs::*;
+    t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
+}
+#[cfg(windows)]
+fn chmod(_path: &Path, _perms: u32) {}
+
+// We have to run a few shell scripts, which choke quite a bit on both `\`
+// characters and on `C:\` paths, so normalize both of them away.
+fn sanitize_sh(path: &Path) -> String {
+    let path = path.to_str().unwrap().replace("\\", "/");
+    return change_drive(&path).unwrap_or(path);
+
+    fn change_drive(s: &str) -> Option<String> {
+        let mut ch = s.chars();
+        let drive = ch.next().unwrap_or('C');
+        if ch.next() != Some(':') {
+            return None
+        }
+        if ch.next() != Some('/') {
+            return None
+        }
+        Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
+    }
+}

src/bootstrap/build/mod.rs

@@ -34,6 +34,7 @@
 mod clean;
 mod compile;
 mod config;
+mod dist;
 mod doc;
 mod flags;
 mod native;
@@ -76,6 +77,7 @@ pub struct Build {
     short_ver_hash: Option<String>,
     ver_date: Option<String>,
     version: String,
+    package_vers: String,
     bootstrap_key: String,
 
     // Runtime state filled in later on
@@ -121,6 +123,7 @@ pub fn new(flags: Flags, config: Config) -> Build {
             ver_date: None,
             version: String::new(),
             bootstrap_key: String::new(),
+            package_vers: String::new(),
             cc: HashMap::new(),
             cxx: HashMap::new(),
             compiler_rt_built: RefCell::new(HashMap::new()),
@@ -208,6 +211,12 @@ pub fn build(&mut self) {
                     check::linkcheck(self, stage, target.target);
                 }
 
+                DistDocs { stage } => dist::docs(self, stage, target.target),
+                DistMingw { _dummy } => dist::mingw(self, target.target),
+                DistRustc { stage } => dist::rustc(self, stage, target.target),
+                DistStd { compiler } => dist::std(self, &compiler, target.target),
+
+                Dist { .. } |
                 Doc { .. } | // pseudo-steps
                 Check { .. } => {}
             }

src/bootstrap/build/step.rs

@@ -73,6 +73,13 @@ pub struct Step<'a> {
             // target to depend on a bunch of others.
             (check, Check { stage: u32, compiler: Compiler<'a> }),
             (check_linkcheck, CheckLinkcheck { stage: u32 }),
+
+            // Distribution targets, creating tarballs
+            (dist, Dist { stage: u32 }),
+            (dist_docs, DistDocs { stage: u32 }),
+            (dist_mingw, DistMingw { _dummy: () }),
+            (dist_rustc, DistRustc { stage: u32 }),
+            (dist_std, DistStd { compiler: Compiler<'a> }),
         }
     }
 }
@@ -279,7 +286,8 @@ pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
                      self.doc_error_index(stage)]
             }
             Source::Check { stage, compiler: _ } => {
-                vec![self.check_linkcheck(stage)]
+                vec![self.check_linkcheck(stage),
+                     self.dist(stage)]
             }
             Source::CheckLinkcheck { stage } => {
                 vec![self.tool_linkchecker(stage), self.doc(stage)]
@@ -292,6 +300,34 @@ pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
             Source::ToolRustbook { stage } => {
                 vec![self.librustc(self.compiler(stage))]
             }
+
+            Source::DistDocs { stage } => vec![self.doc(stage)],
+            Source::DistMingw { _dummy: _ } => Vec::new(),
+            Source::DistRustc { stage } => {
+                vec![self.rustc(stage)]
+            }
+            Source::DistStd { compiler } => {
+                vec![self.libstd(compiler)]
+            }
+
+            Source::Dist { stage } => {
+                let mut base = Vec::new();
+                base.push(self.dist_docs(stage));
+
+                for host in build.config.host.iter() {
+                    let host = self.target(host);
+                    base.push(host.dist_rustc(stage));
+                    if host.target.contains("windows-gnu") {
+                        base.push(host.dist_mingw(()));
+                    }
+
+                    let compiler = self.compiler(stage);
+                    for target in build.config.target.iter() {
+                        base.push(self.target(target).dist_std(compiler));
+                    }
+                }
+                return base
+            }
         }
     }
 }

src/bootstrap/mk/Makefile.in

@@ -38,3 +38,7 @@ standalone-docs:
 	$(Q)$(BOOTSTRAP) --step doc-standalone
 check:
 	$(Q)$(BOOTSTRAP) --step check
+dist:
+	$(Q)$(BOOTSTRAP) --step dist
+
+.PHONY: dist

src/rustc/Cargo.lock

@@ -78,6 +78,7 @@ dependencies = [
  "rbml 0.0.0",
  "rustc_back 0.0.0",
  "rustc_bitflags 0.0.0",
+ "rustc_const_eval 0.0.0",
  "rustc_data_structures 0.0.0",
  "rustc_front 0.0.0",
  "rustc_llvm 0.0.0",
@@ -111,6 +112,15 @@ dependencies = [
  "syntax 0.0.0",
 ]
 
+[[package]]
+name = "rustc_const_eval"
+version = "0.0.0"
+dependencies = [
+ "log 0.0.0",
+ "serialize 0.0.0",
+ "syntax 0.0.0",
+]
+
 [[package]]
 name = "rustc_data_structures"
 version = "0.0.0"
@@ -187,6 +197,7 @@ dependencies = [
  "rustc 0.0.0",
  "rustc_back 0.0.0",
  "rustc_bitflags 0.0.0",
+ "rustc_const_eval 0.0.0",
  "rustc_front 0.0.0",
  "rustc_llvm 0.0.0",
  "serialize 0.0.0",
@@ -201,6 +212,7 @@ dependencies = [
  "log 0.0.0",
  "rustc 0.0.0",
  "rustc_back 0.0.0",
+ "rustc_const_eval 0.0.0",
  "rustc_data_structures 0.0.0",
  "rustc_front 0.0.0",
  "syntax 0.0.0",
@@ -230,6 +242,7 @@ version = "0.0.0"
 dependencies = [
  "log 0.0.0",
  "rustc 0.0.0",
+ "rustc_back 0.0.0",
  "rustc_bitflags 0.0.0",
  "rustc_front 0.0.0",
  "rustc_metadata 0.0.0",
@@ -270,6 +283,7 @@ dependencies = [
  "log 0.0.0",
  "rustc 0.0.0",
  "rustc_back 0.0.0",
+ "rustc_const_eval 0.0.0",
  "rustc_data_structures 0.0.0",
  "rustc_front 0.0.0",
  "rustc_llvm 0.0.0",
@@ -288,6 +302,7 @@ dependencies = [
  "log 0.0.0",
  "rustc 0.0.0",
  "rustc_back 0.0.0",
+ "rustc_const_eval 0.0.0",
  "rustc_front 0.0.0",
  "rustc_platform_intrinsics 0.0.0",
  "syntax 0.0.0",