//! Serialized configuration of a build. //! //! This module implements parsing `config.toml` configuration files to tweak //! how the build runs. use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; use std::process; use crate::cache::{Interned, INTERNER}; use crate::flags::Flags; pub use crate::flags::Subcommand; use build_helper::t; use serde::Deserialize; /// Global configuration for the entire build and/or bootstrap. /// /// This structure is derived from a combination of both `config.toml` and /// `config.mk`. As of the time of this writing it's unlikely that `config.toml` /// is used all that much, so this is primarily filled out by `config.mk` which /// is generated from `./configure`. /// /// Note that this structure is not decoded directly into, but rather it is /// filled out from the decoded forms of the structs below. For documentation /// each field, see the corresponding fields in /// `config.toml.example`. #[derive(Default)] pub struct Config { pub ccache: Option, pub ninja: bool, pub verbose: usize, pub submodules: bool, pub fast_submodules: bool, pub compiler_docs: bool, pub docs: bool, pub locked_deps: bool, pub vendor: bool, pub target_config: HashMap, Target>, pub full_bootstrap: bool, pub extended: bool, pub tools: Option>, pub sanitizers: bool, pub profiler: bool, pub ignore_git: bool, pub exclude: Vec, pub rustc_error_format: Option, pub json_output: bool, pub test_compare_mode: bool, pub llvm_libunwind: bool, pub skip_only_host_steps: bool, pub on_fail: Option, pub stage: Option, pub keep_stage: Vec, pub src: PathBuf, pub jobs: Option, pub cmd: Subcommand, pub incremental: bool, pub dry_run: bool, pub deny_warnings: bool, pub backtrace_on_ice: bool, // llvm codegen options pub llvm_skip_rebuild: bool, pub llvm_assertions: bool, pub llvm_optimize: bool, pub llvm_thin_lto: bool, pub llvm_release_debuginfo: bool, pub llvm_version_check: bool, pub llvm_static_stdcpp: bool, pub llvm_link_shared: bool, pub llvm_clang_cl: Option, pub llvm_targets: Option, pub llvm_experimental_targets: Option, pub llvm_link_jobs: Option, pub llvm_version_suffix: Option, pub llvm_use_linker: Option, pub llvm_allow_old_toolchain: Option, pub use_lld: bool, pub lld_enabled: bool, pub llvm_tools_enabled: bool, pub llvm_cflags: Option, pub llvm_cxxflags: Option, pub llvm_ldflags: Option, pub llvm_use_libcxx: bool, // rust codegen options pub rust_optimize: bool, pub rust_codegen_units: Option, pub rust_codegen_units_std: Option, pub rust_debug_assertions: bool, pub rust_debuginfo_level_rustc: u32, pub rust_debuginfo_level_std: u32, pub rust_debuginfo_level_tools: u32, pub rust_debuginfo_level_tests: u32, pub rust_rpath: bool, pub rustc_parallel: bool, pub rustc_default_linker: Option, pub rust_optimize_tests: bool, pub rust_dist_src: bool, pub rust_codegen_backends: Vec>, pub rust_verify_llvm_ir: bool, pub rust_thin_lto_import_instr_limit: Option, pub rust_remap_debuginfo: bool, pub build: Interned, pub hosts: Vec>, pub targets: Vec>, pub local_rebuild: bool, pub jemalloc: bool, pub control_flow_guard: bool, // dist misc pub dist_sign_folder: Option, pub dist_upload_addr: Option, pub dist_gpg_password_file: Option, // libstd features pub backtrace: bool, // support for RUST_BACKTRACE // misc pub low_priority: bool, pub channel: String, pub verbose_tests: bool, pub save_toolstates: Option, pub print_step_timings: bool, pub missing_tools: bool, // Fallback musl-root for all targets pub musl_root: Option, pub prefix: Option, pub sysconfdir: Option, pub datadir: Option, pub docdir: Option, pub bindir: PathBuf, pub libdir: Option, pub mandir: Option, pub codegen_tests: bool, pub nodejs: Option, pub gdb: Option, pub python: Option, pub cargo_native_static: bool, pub configure_args: Vec, // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, pub initial_rustfmt: Option, pub out: PathBuf, } /// Per-target configuration stored in the global configuration structure. #[derive(Default)] pub struct Target { /// Some(path to llvm-config) if using an external LLVM. pub llvm_config: Option, /// Some(path to FileCheck) if one was specified. pub llvm_filecheck: Option, pub cc: Option, pub cxx: Option, pub ar: Option, pub ranlib: Option, pub linker: Option, pub ndk: Option, pub crt_static: Option, pub musl_root: Option, pub wasi_root: Option, pub qemu_rootfs: Option, pub no_std: bool, } impl Target { pub fn from_triple(triple: &str) -> Self { let mut target: Self = Default::default(); if triple.contains("-none") || triple.contains("nvptx") { target.no_std = true; } target } } /// Structure of the `config.toml` file that configuration is read from. /// /// This structure uses `Decodable` to automatically decode a TOML configuration /// file into this format, and then this is traversed and written into the above /// `Config` structure. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct TomlConfig { build: Option, install: Option, llvm: Option, rust: Option, target: Option>, dist: Option, } /// TOML representation of various global build decisions. #[derive(Deserialize, Default, Clone)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Build { build: Option, #[serde(default)] host: Vec, #[serde(default)] target: Vec, // This is ignored, the rust code always gets the build directory from the `BUILD_DIR` env variable build_dir: Option, cargo: Option, rustc: Option, rustfmt: Option, /* allow bootstrap.py to use rustfmt key */ docs: Option, compiler_docs: Option, submodules: Option, fast_submodules: Option, gdb: Option, nodejs: Option, python: Option, locked_deps: Option, vendor: Option, full_bootstrap: Option, extended: Option, tools: Option>, verbose: Option, sanitizers: Option, profiler: Option, cargo_native_static: Option, low_priority: Option, configure_args: Option>, local_rebuild: Option, print_step_timings: Option, } /// TOML representation of various global install decisions. #[derive(Deserialize, Default, Clone)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Install { prefix: Option, sysconfdir: Option, docdir: Option, bindir: Option, libdir: Option, mandir: Option, datadir: Option, // standard paths, currently unused infodir: Option, localstatedir: Option, } /// TOML representation of how the LLVM build is configured. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Llvm { skip_rebuild: Option, optimize: Option, thin_lto: Option, release_debuginfo: Option, assertions: Option, ccache: Option, version_check: Option, static_libstdcpp: Option, ninja: Option, targets: Option, experimental_targets: Option, link_jobs: Option, link_shared: Option, version_suffix: Option, clang_cl: Option, cflags: Option, cxxflags: Option, ldflags: Option, use_libcxx: Option, use_linker: Option, allow_old_toolchain: Option, } #[derive(Deserialize, Default, Clone)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Dist { sign_folder: Option, gpg_password_file: Option, upload_addr: Option, src_tarball: Option, missing_tools: Option, } #[derive(Deserialize)] #[serde(untagged)] enum StringOrBool { String(String), Bool(bool), } impl Default for StringOrBool { fn default() -> StringOrBool { StringOrBool::Bool(false) } } /// TOML representation of how the Rust build is configured. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Rust { optimize: Option, debug: Option, codegen_units: Option, codegen_units_std: Option, debug_assertions: Option, debuginfo_level: Option, debuginfo_level_rustc: Option, debuginfo_level_std: Option, debuginfo_level_tools: Option, debuginfo_level_tests: Option, backtrace: Option, incremental: Option, parallel_compiler: Option, default_linker: Option, channel: Option, musl_root: Option, rpath: Option, verbose_tests: Option, optimize_tests: Option, codegen_tests: Option, ignore_git: Option, dist_src: Option, save_toolstates: Option, codegen_backends: Option>, lld: Option, use_lld: Option, llvm_tools: Option, deny_warnings: Option, backtrace_on_ice: Option, verify_llvm_ir: Option, thin_lto_import_instr_limit: Option, remap_debuginfo: Option, jemalloc: Option, test_compare_mode: Option, llvm_libunwind: Option, control_flow_guard: Option, } /// TOML representation of how each build target is configured. #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct TomlTarget { cc: Option, cxx: Option, ar: Option, ranlib: Option, linker: Option, llvm_config: Option, llvm_filecheck: Option, android_ndk: Option, crt_static: Option, musl_root: Option, wasi_root: Option, qemu_rootfs: Option, no_std: Option, } impl Config { fn path_from_python(var_key: &str) -> PathBuf { match env::var_os(var_key) { Some(var_val) => Self::normalize_python_path(var_val), _ => panic!("expected '{}' to be set", var_key), } } /// Normalizes paths from Python slightly. We don't trust paths from Python (#49785). fn normalize_python_path(path: OsString) -> PathBuf { Path::new(&path).components().collect() } pub fn default_opts() -> Config { let mut config = Config::default(); config.llvm_optimize = true; config.llvm_version_check = true; config.backtrace = true; config.rust_optimize = true; config.rust_optimize_tests = true; config.submodules = true; config.fast_submodules = true; config.docs = true; config.rust_rpath = true; config.channel = "dev".to_string(); config.codegen_tests = true; config.ignore_git = false; config.rust_dist_src = true; config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; config.deny_warnings = true; config.missing_tools = false; // set by bootstrap.py config.build = INTERNER.intern_str(&env::var("BUILD").expect("'BUILD' to be set")); config.src = Config::path_from_python("SRC"); config.out = Config::path_from_python("BUILD_DIR"); config.initial_rustc = Config::path_from_python("RUSTC"); config.initial_cargo = Config::path_from_python("CARGO"); config.initial_rustfmt = env::var_os("RUSTFMT").map(Config::normalize_python_path); config } pub fn parse(args: &[String]) -> Config { let flags = Flags::parse(&args); let file = flags.config.clone(); let mut config = Config::default_opts(); config.exclude = flags.exclude; config.rustc_error_format = flags.rustc_error_format; config.json_output = flags.json_output; config.on_fail = flags.on_fail; config.stage = flags.stage; config.jobs = flags.jobs.map(threads_from_config); config.cmd = flags.cmd; config.incremental = flags.incremental; config.dry_run = flags.dry_run; config.keep_stage = flags.keep_stage; config.bindir = "bin".into(); // default if let Some(value) = flags.deny_warnings { config.deny_warnings = value; } if config.dry_run { let dir = config.out.join("tmp-dry-run"); t!(fs::create_dir_all(&dir)); config.out = dir; } // If --target was specified but --host wasn't specified, don't run any host-only tests. let has_hosts = !flags.host.is_empty(); let has_targets = !flags.target.is_empty(); config.skip_only_host_steps = !has_hosts && has_targets; let toml = file .map(|file| { let contents = t!(fs::read_to_string(&file)); match toml::from_str(&contents) { Ok(table) => table, Err(err) => { println!( "failed to parse TOML configuration '{}': {}", file.display(), err ); process::exit(2); } } }) .unwrap_or_else(TomlConfig::default); let build = toml.build.clone().unwrap_or_default(); // set by bootstrap.py config.hosts.push(config.build.clone()); for host in build.host.iter() { let host = INTERNER.intern_str(host); if !config.hosts.contains(&host) { config.hosts.push(host); } } for target in config.hosts.iter().cloned().chain(build.target.iter().map(|s| INTERNER.intern_str(s))) { if !config.targets.contains(&target) { config.targets.push(target); } } config.hosts = if !flags.host.is_empty() { flags.host } else { config.hosts }; config.targets = if !flags.target.is_empty() { flags.target } else { config.targets }; config.nodejs = build.nodejs.map(PathBuf::from); config.gdb = build.gdb.map(PathBuf::from); config.python = build.python.map(PathBuf::from); set(&mut config.low_priority, build.low_priority); set(&mut config.compiler_docs, build.compiler_docs); set(&mut config.docs, build.docs); set(&mut config.submodules, build.submodules); set(&mut config.fast_submodules, build.fast_submodules); set(&mut config.locked_deps, build.locked_deps); set(&mut config.vendor, build.vendor); set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); config.tools = build.tools; set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); set(&mut config.profiler, build.profiler); set(&mut config.cargo_native_static, build.cargo_native_static); set(&mut config.configure_args, build.configure_args); set(&mut config.local_rebuild, build.local_rebuild); set(&mut config.print_step_timings, build.print_step_timings); config.verbose = cmp::max(config.verbose, flags.verbose); if let Some(ref install) = toml.install { config.prefix = install.prefix.clone().map(PathBuf::from); config.sysconfdir = install.sysconfdir.clone().map(PathBuf::from); config.datadir = install.datadir.clone().map(PathBuf::from); config.docdir = install.docdir.clone().map(PathBuf::from); set(&mut config.bindir, install.bindir.clone().map(PathBuf::from)); config.libdir = install.libdir.clone().map(PathBuf::from); config.mandir = install.mandir.clone().map(PathBuf::from); } // We want the llvm-skip-rebuild flag to take precedence over the // skip-rebuild config.toml option so we store it separately // so that we can infer the right value let mut llvm_skip_rebuild = flags.llvm_skip_rebuild; // Store off these values as options because if they're not provided // we'll infer default values for them later let mut llvm_assertions = None; let mut debug = None; let mut debug_assertions = None; let mut debuginfo_level = None; let mut debuginfo_level_rustc = None; let mut debuginfo_level_std = None; let mut debuginfo_level_tools = None; let mut debuginfo_level_tests = None; let mut optimize = None; let mut ignore_git = None; if let Some(ref llvm) = toml.llvm { match llvm.ccache { Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), Some(StringOrBool::Bool(true)) => { config.ccache = Some("ccache".to_string()); } Some(StringOrBool::Bool(false)) | None => {} } set(&mut config.ninja, llvm.ninja); llvm_assertions = llvm.assertions; llvm_skip_rebuild = llvm_skip_rebuild.or(llvm.skip_rebuild); set(&mut config.llvm_optimize, llvm.optimize); set(&mut config.llvm_thin_lto, llvm.thin_lto); set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); set(&mut config.llvm_version_check, llvm.version_check); set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); set(&mut config.llvm_link_shared, llvm.link_shared); config.llvm_targets = llvm.targets.clone(); config.llvm_experimental_targets = llvm.experimental_targets.clone(); config.llvm_link_jobs = llvm.link_jobs; config.llvm_version_suffix = llvm.version_suffix.clone(); config.llvm_clang_cl = llvm.clang_cl.clone(); config.llvm_cflags = llvm.cflags.clone(); config.llvm_cxxflags = llvm.cxxflags.clone(); config.llvm_ldflags = llvm.ldflags.clone(); set(&mut config.llvm_use_libcxx, llvm.use_libcxx); config.llvm_use_linker = llvm.use_linker.clone(); config.llvm_allow_old_toolchain = llvm.allow_old_toolchain; } if let Some(ref rust) = toml.rust { debug = rust.debug; debug_assertions = rust.debug_assertions; debuginfo_level = rust.debuginfo_level; debuginfo_level_rustc = rust.debuginfo_level_rustc; debuginfo_level_std = rust.debuginfo_level_std; debuginfo_level_tools = rust.debuginfo_level_tools; debuginfo_level_tests = rust.debuginfo_level_tests; optimize = rust.optimize; ignore_git = rust.ignore_git; set(&mut config.rust_optimize_tests, rust.optimize_tests); set(&mut config.codegen_tests, rust.codegen_tests); set(&mut config.rust_rpath, rust.rpath); set(&mut config.jemalloc, rust.jemalloc); set(&mut config.test_compare_mode, rust.test_compare_mode); set(&mut config.llvm_libunwind, rust.llvm_libunwind); set(&mut config.backtrace, rust.backtrace); set(&mut config.channel, rust.channel.clone()); set(&mut config.rust_dist_src, rust.dist_src); set(&mut config.verbose_tests, rust.verbose_tests); // in the case "false" is set explicitly, do not overwrite the command line args if let Some(true) = rust.incremental { config.incremental = true; } set(&mut config.use_lld, rust.use_lld); set(&mut config.lld_enabled, rust.lld); set(&mut config.llvm_tools_enabled, rust.llvm_tools); config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); config.rustc_default_linker = rust.default_linker.clone(); config.musl_root = rust.musl_root.clone().map(PathBuf::from); config.save_toolstates = rust.save_toolstates.clone().map(PathBuf::from); set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings)); set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); set(&mut config.control_flow_guard, rust.control_flow_guard); if let Some(ref backends) = rust.codegen_backends { config.rust_codegen_backends = backends.iter().map(|s| INTERNER.intern_str(s)).collect(); } config.rust_codegen_units = rust.codegen_units.map(threads_from_config); config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); } if let Some(ref t) = toml.target { for (triple, cfg) in t { let mut target = Target::from_triple(triple); if let Some(ref s) = cfg.llvm_config { target.llvm_config = Some(config.src.join(s)); } if let Some(ref s) = cfg.llvm_filecheck { target.llvm_filecheck = Some(config.src.join(s)); } if let Some(ref s) = cfg.android_ndk { target.ndk = Some(config.src.join(s)); } if let Some(s) = cfg.no_std { target.no_std = s; } target.cc = cfg.cc.clone().map(PathBuf::from); target.cxx = cfg.cxx.clone().map(PathBuf::from); target.ar = cfg.ar.clone().map(PathBuf::from); target.ranlib = cfg.ranlib.clone().map(PathBuf::from); target.linker = cfg.linker.clone().map(PathBuf::from); target.crt_static = cfg.crt_static; target.musl_root = cfg.musl_root.clone().map(PathBuf::from); target.wasi_root = cfg.wasi_root.clone().map(PathBuf::from); target.qemu_rootfs = cfg.qemu_rootfs.clone().map(PathBuf::from); config.target_config.insert(INTERNER.intern_string(triple.clone()), target); } } if let Some(ref t) = toml.dist { config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from); config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from); config.dist_upload_addr = t.upload_addr.clone(); set(&mut config.rust_dist_src, t.src_tarball); set(&mut config.missing_tools, t.missing_tools); } // Now that we've reached the end of our configuration, infer the // default values for all options that we haven't otherwise stored yet. set(&mut config.initial_rustc, build.rustc.map(PathBuf::from)); set(&mut config.initial_cargo, build.cargo.map(PathBuf::from)); config.llvm_skip_rebuild = llvm_skip_rebuild.unwrap_or(false); let default = false; config.llvm_assertions = llvm_assertions.unwrap_or(default); let default = true; config.rust_optimize = optimize.unwrap_or(default); let default = debug == Some(true); config.rust_debug_assertions = debug_assertions.unwrap_or(default); let with_defaults = |debuginfo_level_specific: Option| { debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { 2 } else { 0 }) }; config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(0); let default = config.channel == "dev"; config.ignore_git = ignore_git.unwrap_or(default); config } /// Try to find the relative path of `bindir`, otherwise return it in full. pub fn bindir_relative(&self) -> &Path { let bindir = &self.bindir; if bindir.is_absolute() { // Try to make it relative to the prefix. if let Some(prefix) = &self.prefix { if let Ok(stripped) = bindir.strip_prefix(prefix) { return stripped; } } } bindir } /// Try to find the relative path of `libdir`. pub fn libdir_relative(&self) -> Option<&Path> { let libdir = self.libdir.as_ref()?; if libdir.is_relative() { Some(libdir) } else { // Try to make it relative to the prefix. libdir.strip_prefix(self.prefix.as_ref()?).ok() } } pub fn verbose(&self) -> bool { self.verbose > 0 } pub fn very_verbose(&self) -> bool { self.verbose > 1 } pub fn llvm_enabled(&self) -> bool { self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) } } fn set(field: &mut T, val: Option) { if let Some(v) = val { *field = v; } } fn threads_from_config(v: u32) -> u32 { match v { 0 => num_cpus::get() as u32, n => n, } }