Skip to content

R
Rust

int / Rust
大约 1 年 前同步成功

通知 1
Star 0
Fork 0
R
Rust

体验新版 GitCode，发现更多精彩内容 >>

提交 303d9168 编写于 作者: B bors
浏览文件
操作

Auto merge of #96687 - jyn514:download-rustc, r=Mark-Simulacrum 

Move download-rustc from python to rustbuild

- Remove download-rustc handling from bootstrap.py
- Allow a custom `pattern` in `builder.unpack()`
- Only download rustc once another part of bootstrap depends on it.

  This is somewhat necessary since the download functions rely on having a full
  `Builder`, which isn't available until after config parsing finishes.

Helps with https://github.com/rust-lang/rust/issues/94829.
上级 0acc4a35 00bb4df1
隐藏空白更改
内联 并排
Showing with 418 addition and 352 deletion
src/bootstrap/bootstrap.py
浏览文件 @ 303d9168
...@@ -444,9 +444,8 @@ class RustBuild(object): ...@@ -444,9 +444,8 @@ class RustBuild(object):
self.verbose = False self.verbose = False
self.git_version = None self.git_version = None
self.nix_deps_dir = None self.nix_deps_dir = None
self.rustc_commit = None
def download_toolchain(self, stage0=True, rustc_channel=None): def download_toolchain(self):
"""Fetch the build system for Rust, written in Rust """Fetch the build system for Rust, written in Rust
This method will build a cache directory, then it will fetch the This method will build a cache directory, then it will fetch the
...@@ -456,37 +455,27 @@ class RustBuild(object): ...@@ -456,37 +455,27 @@ class RustBuild(object):
Each downloaded tarball is extracted, after that, the script Each downloaded tarball is extracted, after that, the script
will move all the content to the right place. will move all the content to the right place.
""" """
if rustc_channel is None: rustc_channel = self.stage0_compiler.version
rustc_channel = self.stage0_compiler.version bin_root = self.bin_root()
bin_root = self.bin_root(stage0)
key = self.stage0_compiler.date key = self.stage0_compiler.date
if not stage0: if self.rustc().startswith(bin_root) and \
key += str(self.rustc_commit) (not os.path.exists(self.rustc()) or
if self.rustc(stage0).startswith(bin_root) and \ self.program_out_of_date(self.rustc_stamp(), key)):
(not os.path.exists(self.rustc(stage0)) or
self.program_out_of_date(self.rustc_stamp(stage0), key)):
if os.path.exists(bin_root): if os.path.exists(bin_root):
shutil.rmtree(bin_root) shutil.rmtree(bin_root)
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
filename = "rust-std-{}-{}{}".format( filename = "rust-std-{}-{}{}".format(
rustc_channel, self.build, tarball_suffix) rustc_channel, self.build, tarball_suffix)
pattern = "rust-std-{}".format(self.build) pattern = "rust-std-{}".format(self.build)
self._download_component_helper(filename, pattern, tarball_suffix, stage0) self._download_component_helper(filename, pattern, tarball_suffix)
filename = "rustc-{}-{}{}".format(rustc_channel, self.build, filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix) tarball_suffix)
self._download_component_helper(filename, "rustc", tarball_suffix, stage0) self._download_component_helper(filename, "rustc", tarball_suffix)
# download-rustc doesn't need its own cargo, it can just use beta's. filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
if stage0: tarball_suffix)
filename = "cargo-{}-{}{}".format(rustc_channel, self.build, self._download_component_helper(filename, "cargo", tarball_suffix)
tarball_suffix) self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
self._download_component_helper(filename, "cargo", tarball_suffix)
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
else:
filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
self._download_component_helper(
filename, "rustc-dev", tarball_suffix, stage0
)
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
...@@ -494,7 +483,7 @@ class RustBuild(object): ...@@ -494,7 +483,7 @@ class RustBuild(object):
for lib in os.listdir(lib_dir): for lib in os.listdir(lib_dir):
if lib.endswith(".so"): if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(lib_dir, lib)) self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
with output(self.rustc_stamp(stage0)) as rust_stamp: with output(self.rustc_stamp()) as rust_stamp:
rust_stamp.write(key) rust_stamp.write(key)
if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
...@@ -518,24 +507,17 @@ class RustBuild(object): ...@@ -518,24 +507,17 @@ class RustBuild(object):
rustfmt_stamp.write(self.stage0_rustfmt.channel()) rustfmt_stamp.write(self.stage0_rustfmt.channel())
def _download_component_helper( def _download_component_helper(
self, filename, pattern, tarball_suffix, stage0=True, key=None self, filename, pattern, tarball_suffix, key=None
): ):
if key is None: if key is None:
if stage0: key = self.stage0_compiler.date
key = self.stage0_compiler.date
else:
key = self.rustc_commit
cache_dst = os.path.join(self.build_dir, "cache") cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, key) rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache): if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache) os.makedirs(rustc_cache)
if stage0: base = self._download_url
base = self._download_url url = "dist/{}".format(key)
url = "dist/{}".format(key)
else:
base = "https://ci-artifacts.rust-lang.org"
url = "rustc-builds/{}".format(self.rustc_commit)
tarball = os.path.join(rustc_cache, filename) tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball): if not os.path.exists(tarball):
get( get(
...@@ -544,9 +526,9 @@ class RustBuild(object): ...@@ -544,9 +526,9 @@ class RustBuild(object):
tarball, tarball,
self.checksums_sha256, self.checksums_sha256,
verbose=self.verbose, verbose=self.verbose,
do_verify=stage0, do_verify=True,
) )
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose) unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
def fix_bin_or_dylib(self, fname): def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker, """Modifies the interpreter section of 'fname' to fix the dynamic linker,
...@@ -644,62 +626,15 @@ class RustBuild(object): ...@@ -644,62 +626,15 @@ class RustBuild(object):
print("warning: failed to call patchelf:", reason) print("warning: failed to call patchelf:", reason)
return return
# If `download-rustc` is set, download the most recent commit with CI artifacts def rustc_stamp(self):
def maybe_download_ci_toolchain(self):
# If `download-rustc` is not set, default to rebuilding.
download_rustc = self.get_toml("download-rustc", section="rust")
if download_rustc is None or download_rustc == "false":
return None
assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
# Handle running from a directory other than the top level
rev_parse = ["git", "rev-parse", "--show-toplevel"]
top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
compiler = "{}/compiler/".format(top_level)
library = "{}/library/".format(top_level)
# Look for a version to compare to based on the current commit.
# Only commits merged by bors will have CI artifacts.
merge_base = [
"git", "rev-list", "--author=bors@rust-lang.org", "-n1",
"--first-parent", "HEAD"
]
commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
if not commit:
print("error: could not find commit hash for downloading rustc")
print("help: maybe your repository history is too shallow?")
print("help: consider disabling `download-rustc`")
print("help: or fetch enough history to include one upstream commit")
exit(1)
# Warn if there were changes to the compiler or standard library since the ancestor commit.
status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
if status != 0:
if download_rustc == "if-unchanged":
if self.verbose:
print("warning: saw changes to compiler/ or library/ since {}; " \
"ignoring `download-rustc`".format(commit))
return None
print("warning: `download-rustc` is enabled, but there are changes to " \
"compiler/ or library/")
if self.verbose:
print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
self.rustc_commit = commit
# FIXME: support downloading artifacts from the beta channel
self.download_toolchain(False, "nightly")
def rustc_stamp(self, stage0):
"""Return the path for .rustc-stamp at the given stage """Return the path for .rustc-stamp at the given stage
>>> rb = RustBuild() >>> rb = RustBuild()
>>> rb.build_dir = "build" >>> rb.build_dir = "build"
>>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp") >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
True
>>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
True True
""" """
return os.path.join(self.bin_root(stage0), '.rustc-stamp') return os.path.join(self.bin_root(), '.rustc-stamp')
def rustfmt_stamp(self): def rustfmt_stamp(self):
"""Return the path for .rustfmt-stamp """Return the path for .rustfmt-stamp
...@@ -709,7 +644,7 @@ class RustBuild(object): ...@@ -709,7 +644,7 @@ class RustBuild(object):
>>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
True True
""" """
return os.path.join(self.bin_root(True), '.rustfmt-stamp') return os.path.join(self.bin_root(), '.rustfmt-stamp')
def program_out_of_date(self, stamp_path, key): def program_out_of_date(self, stamp_path, key):
"""Check if the given program stamp is out of date""" """Check if the given program stamp is out of date"""
...@@ -718,26 +653,21 @@ class RustBuild(object): ...@@ -718,26 +653,21 @@ class RustBuild(object):
with open(stamp_path, 'r') as stamp: with open(stamp_path, 'r') as stamp:
return key != stamp.read() return key != stamp.read()
def bin_root(self, stage0): def bin_root(self):
"""Return the binary root directory for the given stage """Return the binary root directory for the given stage
>>> rb = RustBuild() >>> rb = RustBuild()
>>> rb.build_dir = "build" >>> rb.build_dir = "build"
>>> rb.bin_root(True) == os.path.join("build", "stage0") >>> rb.bin_root() == os.path.join("build", "stage0")
True
>>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
True True
When the 'build' property is given should be a nested directory: When the 'build' property is given should be a nested directory:
>>> rb.build = "devel" >>> rb.build = "devel"
>>> rb.bin_root(True) == os.path.join("build", "devel", "stage0") >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
True True
""" """
if stage0: subdir = "stage0"
subdir = "stage0"
else:
subdir = "ci-rustc"
return os.path.join(self.build_dir, self.build, subdir) return os.path.join(self.build_dir, self.build, subdir)
def get_toml(self, key, section=None): def get_toml(self, key, section=None):
...@@ -785,9 +715,9 @@ class RustBuild(object): ...@@ -785,9 +715,9 @@ class RustBuild(object):
"""Return config path for cargo""" """Return config path for cargo"""
return self.program_config('cargo') return self.program_config('cargo')
def rustc(self, stage0): def rustc(self):
"""Return config path for rustc""" """Return config path for rustc"""
return self.program_config('rustc', stage0) return self.program_config('rustc')
def rustfmt(self): def rustfmt(self):
"""Return config path for rustfmt""" """Return config path for rustfmt"""
...@@ -795,7 +725,7 @@ class RustBuild(object): ...@@ -795,7 +725,7 @@ class RustBuild(object):
return None return None
return self.program_config('rustfmt') return self.program_config('rustfmt')
def program_config(self, program, stage0=True): def program_config(self, program):
"""Return config path for the given program at the given stage """Return config path for the given program at the given stage
>>> rb = RustBuild() >>> rb = RustBuild()
...@@ -803,19 +733,15 @@ class RustBuild(object): ...@@ -803,19 +733,15 @@ class RustBuild(object):
>>> rb.program_config('rustc') >>> rb.program_config('rustc')
'rustc' 'rustc'
>>> rb.config_toml = '' >>> rb.config_toml = ''
>>> cargo_path = rb.program_config('cargo', True) >>> cargo_path = rb.program_config('cargo')
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True), >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
... "bin", "cargo")
True
>>> cargo_path = rb.program_config('cargo', False)
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
... "bin", "cargo") ... "bin", "cargo")
True True
""" """
config = self.get_toml(program) config = self.get_toml(program)
if config: if config:
return os.path.expanduser(config) return os.path.expanduser(config)
return os.path.join(self.bin_root(stage0), "bin", "{}{}".format( return os.path.join(self.bin_root(), "bin", "{}{}".format(
program, self.exe_suffix())) program, self.exe_suffix()))
@staticmethod @staticmethod
...@@ -871,14 +797,14 @@ class RustBuild(object): ...@@ -871,14 +797,14 @@ class RustBuild(object):
if "CARGO_BUILD_TARGET" in env: if "CARGO_BUILD_TARGET" in env:
del env["CARGO_BUILD_TARGET"] del env["CARGO_BUILD_TARGET"]
env["CARGO_TARGET_DIR"] = build_dir env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc(True) env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LD_LIBRARY_PATH"]) \ (os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else "" if "LD_LIBRARY_PATH" in env else ""
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \ (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else "" if "DYLD_LIBRARY_PATH" in env else ""
env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LIBRARY_PATH"]) \ (os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else "" if "LIBRARY_PATH" in env else ""
...@@ -900,7 +826,7 @@ class RustBuild(object): ...@@ -900,7 +826,7 @@ class RustBuild(object):
if self.get_toml("deny-warnings", "rust") != "false": if self.get_toml("deny-warnings", "rust") != "false":
env["RUSTFLAGS"] += " -Dwarnings" env["RUSTFLAGS"] += " -Dwarnings"
env["PATH"] = os.path.join(self.bin_root(True), "bin") + \ env["PATH"] = os.path.join(self.bin_root(), "bin") + \
os.pathsep + env["PATH"] os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()): if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `{}`".format( raise Exception("no cargo executable found at `{}`".format(
...@@ -1171,8 +1097,6 @@ def bootstrap(help_triggered): ...@@ -1171,8 +1097,6 @@ def bootstrap(help_triggered):
# Fetch/build the bootstrap # Fetch/build the bootstrap
build.download_toolchain() build.download_toolchain()
# Download the master compiler if `download-rustc` is set
build.maybe_download_ci_toolchain()
sys.stdout.flush() sys.stdout.flush()
build.ensure_vendored() build.ensure_vendored()
build.build_bootstrap() build.build_bootstrap()
...@@ -1184,8 +1108,6 @@ def bootstrap(help_triggered): ...@@ -1184,8 +1108,6 @@ def bootstrap(help_triggered):
env = os.environ.copy() env = os.environ.copy()
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
env["BOOTSTRAP_PYTHON"] = sys.executable env["BOOTSTRAP_PYTHON"] = sys.executable
if build.rustc_commit is not None:
env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
run(args, env=env, verbose=build.verbose, is_bootstrap=True) run(args, env=env, verbose=build.verbose, is_bootstrap=True)
......
src/bootstrap/builder.rs
浏览文件 @ 303d9168
...@@ -2,13 +2,14 @@ ...@@ -2,13 +2,14 @@
use std::cell::{Cell, RefCell}; use std::cell::{Cell, RefCell};
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::env; use std::env;
use std::ffi::OsStr; use std::ffi::{OsStr, OsString};
use std::fmt::{Debug, Write}; use std::fmt::{Debug, Write};
use std::fs; use std::fs::{self, File};
use std::hash::Hash; use std::hash::Hash;
use std::io::{BufRead, BufReader, ErrorKind};
use std::ops::Deref; use std::ops::Deref;
use std::path::{Component, Path, PathBuf}; use std::path::{Component, Path, PathBuf};
use std::process::Command; use std::process::{Command, Stdio};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use crate::cache::{Cache, Interned, INTERNER}; use crate::cache::{Cache, Interned, INTERNER};
...@@ -29,7 +30,8 @@ ...@@ -29,7 +30,8 @@
pub use crate::Compiler; pub use crate::Compiler;
// FIXME: replace with std::lazy after it gets stabilized and reaches beta // FIXME: replace with std::lazy after it gets stabilized and reaches beta
use once_cell::sync::Lazy; use once_cell::sync::{Lazy, OnceCell};
use xz2::bufread::XzDecoder;
pub struct Builder<'a> { pub struct Builder<'a> {
pub build: &'a Build, pub build: &'a Build,
...@@ -758,6 +760,207 @@ fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { ...@@ -758,6 +760,207 @@ fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) {
StepDescription::run(v, self, paths); StepDescription::run(v, self, paths);
} }
/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
/// or the RPATH section, to fix the dynamic library search path
///
/// This is only required on NixOS and uses the PatchELF utility to
/// change the interpreter/RPATH of ELF executables.
///
/// Please see https://nixos.org/patchelf.html for more information
pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) {
// FIXME: cache NixOS detection?
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
Err(_) => return,
Ok(output) if !output.status.success() => return,
Ok(output) => {
let mut s = output.stdout;
if s.last() == Some(&b'\n') {
s.pop();
}
if s != b"Linux" {
return;
}
}
}
// If the user has asked binaries to be patched for Nix, then
// don't check for NixOS or `/lib`, just continue to the patching.
// NOTE: this intentionally comes after the Linux check:
// - patchelf only works with ELF files, so no need to run it on Mac or Windows
// - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
if !self.config.patch_binaries_for_nix {
// Use `/etc/os-release` instead of `/etc/NIXOS`.
// The latter one does not exist on NixOS when using tmpfs as root.
const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
let os_release = match File::open("/etc/os-release") {
Err(e) if e.kind() == ErrorKind::NotFound => return,
Err(e) => panic!("failed to access /etc/os-release: {}", e),
Ok(f) => f,
};
if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
return;
}
if Path::new("/lib").exists() {
return;
}
}
// At this point we're pretty sure the user is running NixOS or using Nix
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
// Only build `.nix-deps` once.
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
let mut nix_build_succeeded = true;
let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
// Run `nix-build` to "build" each dependency (which will likely reuse
// the existing `/nix/store` copy, or at most download a pre-built copy).
//
// Importantly, we create a gc-root called `.nix-deps` in the `build/`
// directory, but still reference the actual `/nix/store` path in the rpath
// as it makes it significantly more robust against changes to the location of
// the `.nix-deps` location.
//
// bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
// zlib: Needed as a system dependency of `libLLVM-*.so`.
// patchelf: Needed for patching ELF binaries (see doc comment above).
let nix_deps_dir = self.out.join(".nix-deps");
const NIX_EXPR: &str = "
with (import <nixpkgs> {});
symlinkJoin {
name = \"rust-stage0-dependencies\";
paths = [
zlib
patchelf
stdenv.cc.bintools
];
}
";
nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[
Path::new("-E"),
Path::new(NIX_EXPR),
Path::new("-o"),
&nix_deps_dir,
]));
nix_deps_dir
});
if !nix_build_succeeded {
return;
}
let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
let rpath_entries = {
// ORIGIN is a relative default, all binary and dynamic libraries we ship
// appear to have this (even when `../lib` is redundant).
// NOTE: there are only two paths here, delimited by a `:`
let mut entries = OsString::from("$ORIGIN/../lib:");
entries.push(t!(fs::canonicalize(nix_deps_dir)));
entries.push("/lib");
entries
};
patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
if !fname.extension().map_or(false, |ext| ext == "so") {
// Finally, set the corret .interp for binaries
let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
// FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
}
self.try_run(patchelf.arg(fname));
}
pub(crate) fn download_component(&self, base: &str, url: &str, dest_path: &Path) {
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
// FIXME: support `do_verify` (only really needed for nightly rustfmt)
self.download_with_retries(&tempfile, &format!("{}/{}", base, url));
t!(std::fs::rename(&tempfile, dest_path));
}
fn download_with_retries(&self, tempfile: &Path, url: &str) {
println!("downloading {}", url);
// Try curl. If that fails and we are on windows, fallback to PowerShell.
let mut curl = Command::new("curl");
curl.args(&[
"-#",
"-y",
"30",
"-Y",
"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
"--connect-timeout",
"30", // timeout if cannot connect within 30 seconds
"--retry",
"3",
"-Sf",
"-o",
]);
curl.arg(tempfile);
curl.arg(url);
if !self.check_run(&mut curl) {
if self.build.build.contains("windows-msvc") {
println!("Fallback to PowerShell");
for _ in 0..3 {
if self.try_run(Command::new("PowerShell.exe").args(&[
"/nologo",
"-Command",
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
&format!(
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
),
])) {
return;
}
println!("\nspurious failure, trying again");
}
}
std::process::exit(1);
}
}
pub(crate) fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
println!("extracting {} to {}", tarball.display(), dst.display());
if !dst.exists() {
t!(fs::create_dir_all(dst));
}
// `tarball` ends with `.tar.xz`; strip that suffix
// example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
let uncompressed_filename =
Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
// decompress the file
let data = t!(File::open(tarball));
let decompressor = XzDecoder::new(BufReader::new(data));
let mut tar = tar::Archive::new(decompressor);
for member in t!(tar.entries()) {
let mut member = t!(member);
let original_path = t!(member.path()).into_owned();
// skip the top-level directory
if original_path == directory_prefix {
continue;
}
let mut short_path = t!(original_path.strip_prefix(directory_prefix));
if !short_path.starts_with(pattern) {
continue;
}
short_path = t!(short_path.strip_prefix(pattern));
let dst_path = dst.join(short_path);
self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
if !t!(member.unpack_in(dst)) {
panic!("path traversal attack ??");
}
let src_path = dst.join(original_path);
if src_path.is_dir() && dst_path.exists() {
continue;
}
t!(fs::rename(src_path, dst_path));
}
t!(fs::remove_dir_all(dst.join(directory_prefix)));
}
/// Obtain a compiler at a given stage and for a given host. Explicitly does /// Obtain a compiler at a given stage and for a given host. Explicitly does
/// not take `Compiler` since all `Compiler` instances are meant to be /// not take `Compiler` since all `Compiler` instances are meant to be
/// obtained through this function, since it ensures that they are valid /// obtained through this function, since it ensures that they are valid
...@@ -819,7 +1022,7 @@ fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> { ...@@ -819,7 +1022,7 @@ fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> {
.join("lib"); .join("lib");
// Avoid deleting the rustlib/ directory we just copied // Avoid deleting the rustlib/ directory we just copied
// (in `impl Step for Sysroot`). // (in `impl Step for Sysroot`).
if !builder.config.download_rustc { if !builder.download_rustc() {
let _ = fs::remove_dir_all(&sysroot); let _ = fs::remove_dir_all(&sysroot);
t!(fs::create_dir_all(&sysroot)); t!(fs::create_dir_all(&sysroot));
} }
...@@ -976,6 +1179,10 @@ pub(crate) fn llvm_link_shared(&self) -> bool { ...@@ -976,6 +1179,10 @@ pub(crate) fn llvm_link_shared(&self) -> bool {
Config::llvm_link_shared(self) Config::llvm_link_shared(self)
} }
pub(crate) fn download_rustc(&self) -> bool {
Config::download_rustc(self)
}
/// Prepares an invocation of `cargo` to be run. /// Prepares an invocation of `cargo` to be run.
/// ///
/// This will create a `Command` that represents a pending execution of /// This will create a `Command` that represents a pending execution of
......
src/bootstrap/compile.rs
浏览文件 @ 303d9168
...@@ -42,8 +42,10 @@ impl Step for Std { ...@@ -42,8 +42,10 @@ impl Step for Std {
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
// When downloading stage1, the standard library has already been copied to the sysroot, so // When downloading stage1, the standard library has already been copied to the sysroot, so
// there's no need to rebuild it. // there's no need to rebuild it.
let download_rustc = run.builder.config.download_rustc; let builder = run.builder;
run.all_krates("test").path("library").default_condition(!download_rustc) run.all_krates("test")
.path("library")
.lazy_default_condition(Box::new(|| !builder.download_rustc()))
} }
fn make_run(run: RunConfig<'_>) { fn make_run(run: RunConfig<'_>) {
...@@ -66,7 +68,7 @@ fn run(self, builder: &Builder<'_>) { ...@@ -66,7 +68,7 @@ fn run(self, builder: &Builder<'_>) {
// Don't recompile them. // Don't recompile them.
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
// so its artifacts can't be reused. // so its artifacts can't be reused.
if builder.config.download_rustc && compiler.stage != 0 { if builder.download_rustc() && compiler.stage != 0 {
return; return;
} }
...@@ -551,7 +553,7 @@ fn run(self, builder: &Builder<'_>) { ...@@ -551,7 +553,7 @@ fn run(self, builder: &Builder<'_>) {
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
// so its artifacts can't be reused. // so its artifacts can't be reused.
if builder.config.download_rustc && compiler.stage != 0 { if builder.download_rustc() && compiler.stage != 0 {
// Copy the existing artifacts instead of rebuilding them. // Copy the existing artifacts instead of rebuilding them.
// NOTE: this path is only taken for tools linking to rustc-dev. // NOTE: this path is only taken for tools linking to rustc-dev.
builder.ensure(Sysroot { compiler }); builder.ensure(Sysroot { compiler });
...@@ -995,7 +997,7 @@ fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> { ...@@ -995,7 +997,7 @@ fn run(self, builder: &Builder<'_>) -> Interned<PathBuf> {
t!(fs::create_dir_all(&sysroot)); t!(fs::create_dir_all(&sysroot));
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
if builder.config.download_rustc && compiler.stage != 0 { if builder.download_rustc() && compiler.stage != 0 {
assert_eq!( assert_eq!(
builder.config.build, compiler.host, builder.config.build, compiler.host,
"Cross-compiling is not yet supported with `download-rustc`", "Cross-compiling is not yet supported with `download-rustc`",
...@@ -1090,7 +1092,7 @@ fn run(self, builder: &Builder<'_>) -> Compiler { ...@@ -1090,7 +1092,7 @@ fn run(self, builder: &Builder<'_>) -> Compiler {
let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
if builder.config.download_rustc { if builder.download_rustc() {
builder.ensure(Sysroot { compiler: target_compiler }); builder.ensure(Sysroot { compiler: target_compiler });
return target_compiler; return target_compiler;
} }
......
src/bootstrap/config.rs
浏览文件 @ 303d9168
...@@ -7,9 +7,11 @@ ...@@ -7,9 +7,11 @@
use std::cmp; use std::cmp;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::env; use std::env;
use std::ffi::OsStr;
use std::fmt; use std::fmt;
use std::fs; use std::fs;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::{exit, Command};
use std::str::FromStr; use std::str::FromStr;
use crate::builder::{Builder, TaskPath}; use crate::builder::{Builder, TaskPath};
...@@ -17,7 +19,8 @@ ...@@ -17,7 +19,8 @@
use crate::channel::GitInfo; use crate::channel::GitInfo;
pub use crate::flags::Subcommand; pub use crate::flags::Subcommand;
use crate::flags::{Color, Flags}; use crate::flags::{Color, Flags};
use crate::util::{exe, t}; use crate::util::{exe, output, program_out_of_date, t};
use once_cell::sync::OnceCell;
use serde::{Deserialize, Deserializer}; use serde::{Deserialize, Deserializer};
macro_rules! check_ci_llvm { macro_rules! check_ci_llvm {
...@@ -81,7 +84,11 @@ pub struct Config { ...@@ -81,7 +84,11 @@ pub struct Config {
pub cmd: Subcommand, pub cmd: Subcommand,
pub incremental: bool, pub incremental: bool,
pub dry_run: bool, pub dry_run: bool,
pub download_rustc: bool, /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should.
#[cfg(not(test))]
download_rustc_commit: Option<String>,
#[cfg(test)]
pub download_rustc_commit: Option<String>,
pub deny_warnings: bool, pub deny_warnings: bool,
pub backtrace_on_ice: bool, pub backtrace_on_ice: bool,
...@@ -1080,7 +1087,8 @@ pub fn parse(args: &[String]) -> Config { ...@@ -1080,7 +1087,8 @@ pub fn parse(args: &[String]) -> Config {
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
config.download_rustc = env::var("BOOTSTRAP_DOWNLOAD_RUSTC").as_deref() == Ok("1"); config.download_rustc_commit =
download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
} else { } else {
config.rust_profile_use = flags.rust_profile_use; config.rust_profile_use = flags.rust_profile_use;
config.rust_profile_generate = flags.rust_profile_generate; config.rust_profile_generate = flags.rust_profile_generate;
...@@ -1192,7 +1200,7 @@ pub fn parse(args: &[String]) -> Config { ...@@ -1192,7 +1200,7 @@ pub fn parse(args: &[String]) -> Config {
let default = config.channel == "dev"; let default = config.channel == "dev";
config.ignore_git = ignore_git.unwrap_or(default); config.ignore_git = ignore_git.unwrap_or(default);
let download_rustc = config.download_rustc; let download_rustc = config.download_rustc_commit.is_some();
// See https://github.com/rust-lang/compiler-team/issues/326 // See https://github.com/rust-lang/compiler-team/issues/326
config.stage = match config.cmd { config.stage = match config.cmd {
Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0),
...@@ -1309,6 +1317,23 @@ pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool { ...@@ -1309,6 +1317,23 @@ pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
llvm_link_shared llvm_link_shared
} }
/// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source.
pub(crate) fn download_rustc(builder: &Builder<'_>) -> bool {
static DOWNLOAD_RUSTC: OnceCell<bool> = OnceCell::new();
if builder.config.dry_run && DOWNLOAD_RUSTC.get().is_none() {
// avoid trying to actually download the commit
return false;
}
*DOWNLOAD_RUSTC.get_or_init(|| match &builder.config.download_rustc_commit {
None => false,
Some(commit) => {
download_ci_rustc(builder, commit);
true
}
})
}
pub fn verbose(&self) -> bool { pub fn verbose(&self) -> bool {
self.verbose > 0 self.verbose > 0
} }
...@@ -1358,3 +1383,116 @@ fn threads_from_config(v: u32) -> u32 { ...@@ -1358,3 +1383,116 @@ fn threads_from_config(v: u32) -> u32 {
n => n, n => n,
} }
} }
/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
// If `download-rustc` is not set, default to rebuilding.
let if_unchanged = match download_rustc {
None | Some(StringOrBool::Bool(false)) => return None,
Some(StringOrBool::Bool(true)) => false,
Some(StringOrBool::String(s)) if s == "if-unchanged" => true,
Some(StringOrBool::String(other)) => {
panic!("unrecognized option for download-rustc: {}", other)
}
};
// Handle running from a directory other than the top level
let top_level = output(Command::new("git").args(&["rev-parse", "--show-toplevel"]));
let top_level = top_level.trim_end();
let compiler = format!("{top_level}/compiler/");
let library = format!("{top_level}/library/");
// Look for a version to compare to based on the current commit.
// Only commits merged by bors will have CI artifacts.
let merge_base = output(Command::new("git").args(&[
"rev-list",
"--author=bors@rust-lang.org",
"-n1",
"--first-parent",
"HEAD",
]));
let commit = merge_base.trim_end();
if commit.is_empty() {
println!("error: could not find commit hash for downloading rustc");
println!("help: maybe your repository history is too shallow?");
println!("help: consider disabling `download-rustc`");
println!("help: or fetch enough history to include one upstream commit");
exit(1);
}
// Warn if there were changes to the compiler or standard library since the ancestor commit.
let has_changes = !t!(Command::new("git")
.args(&["diff-index", "--quiet", &commit, "--", &compiler, &library])
.status())
.success();
if has_changes {
if if_unchanged {
if verbose {
println!(
"warning: saw changes to compiler/ or library/ since {commit}; \
ignoring `download-rustc`"
);
}
return None;
}
println!(
"warning: `download-rustc` is enabled, but there are changes to \
compiler/ or library/"
);
}
Some(commit.to_string())
}
fn download_ci_rustc(builder: &Builder<'_>, commit: &str) {
builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
// FIXME: support downloading artifacts from the beta channel
const CHANNEL: &str = "nightly";
let host = builder.config.build.triple;
let bin_root = builder.out.join(host).join("ci-rustc");
let rustc_stamp = bin_root.join(".rustc-stamp");
if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) {
if bin_root.exists() {
t!(fs::remove_dir_all(&bin_root));
}
let filename = format!("rust-std-{CHANNEL}-{host}.tar.xz");
let pattern = format!("rust-std-{host}");
download_component(builder, filename, &pattern, commit);
let filename = format!("rustc-{CHANNEL}-{host}.tar.xz");
download_component(builder, filename, "rustc", commit);
// download-rustc doesn't need its own cargo, it can just use beta's.
let filename = format!("rustc-dev-{CHANNEL}-{host}.tar.xz");
download_component(builder, filename, "rustc-dev", commit);
builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
let lib_dir = bin_root.join("lib");
for lib in t!(fs::read_dir(lib_dir)) {
let lib = t!(lib);
if lib.path().extension() == Some(OsStr::new("so")) {
builder.fix_bin_or_dylib(&lib.path());
}
}
t!(fs::write(rustc_stamp, commit));
}
}
/// Download a single component of a CI-built toolchain (not necessarily a published nightly).
// NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
fn download_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) {
let cache_dst = builder.out.join("cache");
let rustc_cache = cache_dst.join(commit);
if !rustc_cache.exists() {
t!(fs::create_dir_all(&rustc_cache));
}
let base = "https://ci-artifacts.rust-lang.org";
let url = format!("rustc-builds/{commit}");
let tarball = rustc_cache.join(&filename);
if !tarball.exists() {
builder.download_component(base, &format!("{url}/{filename}"), &tarball);
}
let bin_root = builder.out.join(builder.config.build.triple).join("ci-rustc");
builder.unpack(&tarball, &bin_root, prefix)
}
src/bootstrap/native.rs
浏览文件 @ 303d9168
...@@ -12,16 +12,13 @@ ...@@ -12,16 +12,13 @@
use std::env::consts::EXE_EXTENSION; use std::env::consts::EXE_EXTENSION;
use std::ffi::{OsStr, OsString}; use std::ffi::{OsStr, OsString};
use std::fs::{self, File}; use std::fs::{self, File};
use std::io::{self, BufRead, BufReader, ErrorKind}; use std::io;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::{Command, Stdio}; use std::process::Command;
use once_cell::sync::OnceCell;
use xz2::bufread::XzDecoder;
use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::builder::{Builder, RunConfig, ShouldRun, Step};
use crate::config::TargetSelection; use crate::config::TargetSelection;
use crate::util::{self, exe, output, t, up_to_date}; use crate::util::{self, exe, output, program_out_of_date, t, up_to_date};
use crate::{CLang, GitRepo}; use crate::{CLang, GitRepo};
pub struct Meta { pub struct Meta {
...@@ -151,13 +148,13 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { ...@@ -151,13 +148,13 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
download_ci_llvm(builder, &llvm_sha); download_ci_llvm(builder, &llvm_sha);
for binary in ["llvm-config", "FileCheck"] { for binary in ["llvm-config", "FileCheck"] {
fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary)); builder.fix_bin_or_dylib(&llvm_root.join("bin").join(binary));
} }
let llvm_lib = llvm_root.join("lib"); let llvm_lib = llvm_root.join("lib");
for entry in t!(fs::read_dir(&llvm_lib)) { for entry in t!(fs::read_dir(&llvm_lib)) {
let lib = t!(entry).path(); let lib = t!(entry).path();
if lib.extension().map_or(false, |ext| ext == "so") { if lib.extension().map_or(false, |ext| ext == "so") {
fix_bin_or_dylib(builder, &lib); builder.fix_bin_or_dylib(&lib);
} }
} }
t!(fs::write(llvm_stamp, key)); t!(fs::write(llvm_stamp, key));
...@@ -182,218 +179,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { ...@@ -182,218 +179,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple); let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
let tarball = rustc_cache.join(&filename); let tarball = rustc_cache.join(&filename);
if !tarball.exists() { if !tarball.exists() {
download_component(builder, base, &format!("{}/{}", url, filename), &tarball); builder.download_component(base, &format!("{}/{}", url, filename), &tarball);
} }
let llvm_root = builder.config.ci_llvm_root(); let llvm_root = builder.config.ci_llvm_root();
unpack(builder, &tarball, &llvm_root); builder.unpack(&tarball, &llvm_root, "rust-dev");
}
/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
/// or the RPATH section, to fix the dynamic library search path
///
/// This is only required on NixOS and uses the PatchELF utility to
/// change the interpreter/RPATH of ELF executables.
///
/// Please see https://nixos.org/patchelf.html for more information
fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) {
// FIXME: cache NixOS detection?
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
Err(_) => return,
Ok(output) if !output.status.success() => return,
Ok(output) => {
let mut s = output.stdout;
if s.last() == Some(&b'\n') {
s.pop();
}
if s != b"Linux" {
return;
}
}
}
// If the user has asked binaries to be patched for Nix, then
// don't check for NixOS or `/lib`, just continue to the patching.
// FIXME: shouldn't this take precedence over the `uname` check above?
if !builder.config.patch_binaries_for_nix {
// Use `/etc/os-release` instead of `/etc/NIXOS`.
// The latter one does not exist on NixOS when using tmpfs as root.
const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
let os_release = match File::open("/etc/os-release") {
Err(e) if e.kind() == ErrorKind::NotFound => return,
Err(e) => panic!("failed to access /etc/os-release: {}", e),
Ok(f) => f,
};
if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
return;
}
if Path::new("/lib").exists() {
return;
}
}
// At this point we're pretty sure the user is running NixOS or using Nix
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
// Only build `.nix-deps` once.
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
let mut nix_build_succeeded = true;
let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
// Run `nix-build` to "build" each dependency (which will likely reuse
// the existing `/nix/store` copy, or at most download a pre-built copy).
//
// Importantly, we create a gc-root called `.nix-deps` in the `build/`
// directory, but still reference the actual `/nix/store` path in the rpath
// as it makes it significantly more robust against changes to the location of
// the `.nix-deps` location.
//
// bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
// zlib: Needed as a system dependency of `libLLVM-*.so`.
// patchelf: Needed for patching ELF binaries (see doc comment above).
let nix_deps_dir = builder.out.join(".nix-deps");
const NIX_EXPR: &str = "
with (import <nixpkgs> {});
symlinkJoin {
name = \"rust-stage0-dependencies\";
paths = [
zlib
patchelf
stdenv.cc.bintools
];
}
";
nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[
Path::new("-E"),
Path::new(NIX_EXPR),
Path::new("-o"),
&nix_deps_dir,
]));
nix_deps_dir
});
if !nix_build_succeeded {
return;
}
let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
let rpath_entries = {
// ORIGIN is a relative default, all binary and dynamic libraries we ship
// appear to have this (even when `../lib` is redundant).
// NOTE: there are only two paths here, delimited by a `:`
let mut entries = OsString::from("$ORIGIN/../lib:");
entries.push(t!(fs::canonicalize(nix_deps_dir)));
entries.push("/lib");
entries
};
patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
if !fname.extension().map_or(false, |ext| ext == "so") {
// Finally, set the corret .interp for binaries
let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
// FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
}
builder.try_run(patchelf.arg(fname));
}
fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) {
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
let tempfile = builder.tempdir().join(dest_path.file_name().unwrap());
// FIXME: support `do_verify` (only really needed for nightly rustfmt)
// FIXME: support non-utf8 paths?
download_with_retries(builder, tempfile.to_str().unwrap(), &format!("{}/{}", base, url));
t!(std::fs::rename(&tempfile, dest_path));
}
fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) {
println!("downloading {}", url);
// Try curl. If that fails and we are on windows, fallback to PowerShell.
if !builder.check_run(Command::new("curl").args(&[
"-#",
"-y",
"30",
"-Y",
"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
"--connect-timeout",
"30", // timeout if cannot connect within 30 seconds
"--retry",
"3",
"-Sf",
"-o",
tempfile,
url,
])) {
if builder.build.build.contains("windows-msvc") {
println!("Fallback to PowerShell");
for _ in 0..3 {
if builder.try_run(Command::new("PowerShell.exe").args(&[
"/nologo",
"-Command",
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
&format!(
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
url, tempfile
),
])) {
return;
}
println!("\nspurious failure, trying again");
}
}
std::process::exit(1);
}
}
fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) {
println!("extracting {} to {}", tarball.display(), dst.display());
if !dst.exists() {
t!(fs::create_dir_all(dst));
}
// FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild
const MATCH: &str = "rust-dev";
// `tarball` ends with `.tar.xz`; strip that suffix
// example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
let uncompressed_filename =
Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
// decompress the file
let data = t!(File::open(tarball));
let decompressor = XzDecoder::new(BufReader::new(data));
let mut tar = tar::Archive::new(decompressor);
for member in t!(tar.entries()) {
let mut member = t!(member);
let original_path = t!(member.path()).into_owned();
// skip the top-level directory
if original_path == directory_prefix {
continue;
}
let mut short_path = t!(original_path.strip_prefix(directory_prefix));
if !short_path.starts_with(MATCH) {
continue;
}
short_path = t!(short_path.strip_prefix(MATCH));
let dst_path = dst.join(short_path);
builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
if !t!(member.unpack_in(dst)) {
panic!("path traversal attack ??");
}
let src_path = dst.join(original_path);
if src_path.is_dir() && dst_path.exists() {
continue;
}
t!(fs::rename(src_path, dst_path));
}
t!(fs::remove_dir_all(dst.join(directory_prefix)));
}
fn program_out_of_date(stamp: &Path, key: &str) -> bool {
if !stamp.exists() {
return true;
}
t!(fs::read_to_string(stamp)) != key
} }
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
......
src/bootstrap/test.rs
浏览文件 @ 303d9168
...@@ -2058,7 +2058,7 @@ fn run(self, builder: &Builder<'_>) { ...@@ -2058,7 +2058,7 @@ fn run(self, builder: &Builder<'_>) {
let test_kind = self.test_kind; let test_kind = self.test_kind;
let target = self.host; let target = self.host;
let compiler = if builder.config.download_rustc { let compiler = if builder.download_rustc() {
builder.compiler(builder.top_stage, target) builder.compiler(builder.top_stage, target)
} else { } else {
// Use the previous stage compiler to reuse the artifacts that are // Use the previous stage compiler to reuse the artifacts that are
...@@ -2127,7 +2127,7 @@ fn run(self, builder: &Builder<'_>) { ...@@ -2127,7 +2127,7 @@ fn run(self, builder: &Builder<'_>) {
// with. // with.
// //
// Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution.
let libdir = if builder.config.download_rustc { let libdir = if builder.download_rustc() {
builder.rustc_libdir(compiler) builder.rustc_libdir(compiler)
} else { } else {
builder.sysroot_libdir(compiler, target).to_path_buf() builder.sysroot_libdir(compiler, target).to_path_buf()
......
src/bootstrap/tool.rs
浏览文件 @ 303d9168
...@@ -516,7 +516,7 @@ fn run(self, builder: &Builder<'_>) -> PathBuf { ...@@ -516,7 +516,7 @@ fn run(self, builder: &Builder<'_>) -> PathBuf {
builder.ensure(compile::Rustc { compiler: build_compiler, target: target_compiler.host }); builder.ensure(compile::Rustc { compiler: build_compiler, target: target_compiler.host });
// NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0
// compiler, since you do just as much work. // compiler, since you do just as much work.
if !builder.config.dry_run && builder.config.download_rustc && build_compiler.stage == 0 { if !builder.config.dry_run && builder.download_rustc() && build_compiler.stage == 0 {
println!( println!(
"warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead"
); );
......
src/bootstrap/util.rs
浏览文件 @ 303d9168
...@@ -115,6 +115,14 @@ fn drop(&mut self) { ...@@ -115,6 +115,14 @@ fn drop(&mut self) {
} }
} }
/// Used for download caching
pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool {
if !stamp.exists() {
return true;
}
t!(fs::read_to_string(stamp)) != key
}
/// Symlinks two directories, using junctions on Windows and normal symlinks on /// Symlinks two directories, using junctions on Windows and normal symlinks on
/// Unix. /// Unix.
pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> { pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> {
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或