Auto merge of #43630 - Mark-Simulacrum:rustbuild-cleanups, r=alexcrichton

Rustbuild cleanups/fixes and improvements

Each commit is a standalone change, and can/should be reviewed separately.

This adds two new functionalities:

 - `--target` and `--host` can be passed without changing config.toml, and we'll respect the users' wishes, instead of requiring that all possible targets are passed.
   - Note that this means that `./x.py clean` won't be quite as wide-spread as before, since it limits itself to the configured hosts, not all hosts. This could be considered a feature as well.
 - `ignore-git` field in `config.toml` which tells Rustbuild to not attempt to load git hashes from `.git`.

This is a precursor to eventual further simplification of the configuration system, but I want to get this merged first so that later work can be made in individual PRs.

r? @alexcrichton

Author: bors

config.toml.example

@@ -258,6 +258,9 @@
 # saying that the FileCheck executable is missing, you may want to disable this.
 #codegen-tests = true
 
+# Flag indicating whether git info will be retrieved from .git automatically.
+#ignore-git = false
+
 # =============================================================================
 # Options for specific targets
 #

src/bootstrap/bin/main.rs

@@ -21,11 +21,10 @@ extern crate bootstrap;
 
 use std::env;
 
-use bootstrap::{Flags, Config, Build};
+use bootstrap::{Config, Build};
 
 fn main() {
     let args = env::args().skip(1).collect::<Vec<_>>();
-    let flags = Flags::parse(&args);
-    let config = Config::parse(&flags.build, flags.config.clone());
-    Build::new(flags, config).build();
+    let config = Config::parse(&args);
+    Build::new(config).build();
 }

src/bootstrap/builder.rs

@@ -120,28 +120,19 @@ impl StepDescription {
     fn maybe_run(&self, builder: &Builder, path: Option<&Path>) {
         let build = builder.build;
         let hosts = if self.only_build_targets || self.only_build {
-            &build.config.host[..1]
+            build.build_triple()
         } else {
             &build.hosts
         };
 
-        // Determine the actual targets participating in this rule.
-        // NOTE: We should keep the full projection from build triple to
-        // the hosts for the dist steps, now that the hosts array above is
-        // truncated to avoid duplication of work in that case. Therefore
-        // the original non-shadowed hosts array is used below.
+        // Determine the targets participating in this rule.
         let targets = if self.only_hosts {
-            // If --target was specified but --host wasn't specified,
-            // don't run any host-only tests. Also, respect any `--host`
-            // overrides as done for `hosts`.
-            if build.flags.host.len() > 0 {
-                &build.flags.host[..]
-            } else if build.flags.target.len() > 0 {
+            if build.config.run_host_only {
                 &[]
             } else if self.only_build {
-                &build.config.host[..1]
+                build.build_triple()
             } else {
-                &build.config.host[..]
+                &build.hosts
             }
         } else {
             &build.targets
@@ -288,7 +279,7 @@ impl<'a> Builder<'a> {
 
         let builder = Builder {
             build: build,
-            top_stage: build.flags.stage.unwrap_or(2),
+            top_stage: build.config.stage.unwrap_or(2),
             kind: kind,
             cache: Cache::new(),
             stack: RefCell::new(Vec::new()),
@@ -307,7 +298,7 @@ impl<'a> Builder<'a> {
     }
 
     pub fn run(build: &Build) {
-        let (kind, paths) = match build.flags.cmd {
+        let (kind, paths) = match build.config.cmd {
             Subcommand::Build { ref paths } => (Kind::Build, &paths[..]),
             Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
             Subcommand::Test { ref paths, .. } => (Kind::Test, &paths[..]),
@@ -319,7 +310,7 @@ impl<'a> Builder<'a> {
 
         let builder = Builder {
             build: build,
-            top_stage: build.flags.stage.unwrap_or(2),
+            top_stage: build.config.stage.unwrap_or(2),
             kind: kind,
             cache: Cache::new(),
             stack: RefCell::new(Vec::new()),
@@ -414,22 +405,19 @@ impl<'a> Builder<'a> {
         }
     }
 
-    pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
-        self.ensure(tool::Rustdoc { target_compiler: compiler })
+    pub fn rustdoc(&self, host: Interned<String>) -> PathBuf {
+        self.ensure(tool::Rustdoc { host })
     }
 
-    pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
+    pub fn rustdoc_cmd(&self, host: Interned<String>) -> Command {
         let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
+        let compiler = self.compiler(self.top_stage, host);
         cmd
             .env("RUSTC_STAGE", compiler.stage.to_string())
-            .env("RUSTC_SYSROOT", if compiler.is_snapshot(&self.build) {
-                INTERNER.intern_path(self.build.rustc_snapshot_libdir())
-            } else {
-                self.sysroot(compiler)
-            })
-            .env("RUSTC_LIBDIR", self.rustc_libdir(compiler))
+            .env("RUSTC_SYSROOT", self.sysroot(compiler))
+            .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler, self.build.build))
             .env("CFG_RELEASE_CHANNEL", &self.build.config.channel)
-            .env("RUSTDOC_REAL", self.rustdoc(compiler));
+            .env("RUSTDOC_REAL", self.rustdoc(host));
         cmd
     }
 
@@ -483,7 +471,7 @@ impl<'a> Builder<'a> {
              .env("RUSTC_RPATH", self.config.rust_rpath.to_string())
              .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
              .env("RUSTDOC_REAL", if cmd == "doc" || cmd == "test" {
-                 self.rustdoc(compiler)
+                 self.rustdoc(compiler.host)
              } else {
                  PathBuf::from("/path/to/nowhere/rustdoc/not/required")
              })
@@ -543,12 +531,12 @@ impl<'a> Builder<'a> {
         // Ignore incremental modes except for stage0, since we're
         // not guaranteeing correctness across builds if the compiler
         // is changing under your feet.`
-        if self.flags.incremental && compiler.stage == 0 {
+        if self.config.incremental && compiler.stage == 0 {
             let incr_dir = self.incremental_dir(compiler);
             cargo.env("RUSTC_INCREMENTAL", incr_dir);
         }
 
-        if let Some(ref on_fail) = self.flags.on_fail {
+        if let Some(ref on_fail) = self.config.on_fail {
             cargo.env("RUSTC_ON_FAIL", on_fail);
         }
 

src/bootstrap/cc.rs

@@ -32,6 +32,7 @@
 //! everything.
 
 use std::process::Command;
+use std::iter;
 
 use build_helper::{cc2ar, output};
 use gcc;
@@ -43,47 +44,41 @@ use cache::Interned;
 pub fn find(build: &mut Build) {
     // For all targets we're going to need a C compiler for building some shims
     // and such as well as for being a linker for Rust code.
-    //
-    // This includes targets that aren't necessarily passed on the commandline
-    // (FIXME: Perhaps it shouldn't?)
-    for target in &build.config.target {
+    for target in build.targets.iter().chain(&build.hosts).cloned().chain(iter::once(build.build)) {
         let mut cfg = gcc::Config::new();
         cfg.cargo_metadata(false).opt_level(0).debug(false)
-           .target(target).host(&build.build);
+           .target(&target).host(&build.build);
 
         let config = build.config.target_config.get(&target);
         if let Some(cc) = config.and_then(|c| c.cc.as_ref()) {
             cfg.compiler(cc);
         } else {
-            set_compiler(&mut cfg, "gcc", *target, config, build);
+            set_compiler(&mut cfg, "gcc", target, config, build);
         }
 
         let compiler = cfg.get_compiler();
-        let ar = cc2ar(compiler.path(), target);
-        build.verbose(&format!("CC_{} = {:?}", target, compiler.path()));
+        let ar = cc2ar(compiler.path(), &target);
+        build.verbose(&format!("CC_{} = {:?}", &target, compiler.path()));
         if let Some(ref ar) = ar {
-            build.verbose(&format!("AR_{} = {:?}", target, ar));
+            build.verbose(&format!("AR_{} = {:?}", &target, ar));
         }
-        build.cc.insert(*target, (compiler, ar));
+        build.cc.insert(target, (compiler, ar));
     }
 
     // For all host triples we need to find a C++ compiler as well
-    //
-    // This includes hosts that aren't necessarily passed on the commandline
-    // (FIXME: Perhaps it shouldn't?)
-    for host in &build.config.host {
+    for host in build.hosts.iter().cloned().chain(iter::once(build.build)) {
         let mut cfg = gcc::Config::new();
         cfg.cargo_metadata(false).opt_level(0).debug(false).cpp(true)
-           .target(host).host(&build.build);
-        let config = build.config.target_config.get(host);
+           .target(&host).host(&build.build);
+        let config = build.config.target_config.get(&host);
         if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) {
             cfg.compiler(cxx);
         } else {
-            set_compiler(&mut cfg, "g++", *host, config, build);
+            set_compiler(&mut cfg, "g++", host, config, build);
         }
         let compiler = cfg.get_compiler();
         build.verbose(&format!("CXX_{} = {:?}", host, compiler.path()));
-        build.cxx.insert(*host, compiler);
+        build.cxx.insert(host, compiler);
     }
 }
 

src/bootstrap/channel.rs

@@ -21,6 +21,7 @@ use std::process::Command;
 use build_helper::output;
 
 use Build;
+use config::Config;
 
 // The version number
 pub const CFG_RELEASE_NUM: &str = "1.21.0";
@@ -41,9 +42,9 @@ struct Info {
 }
 
 impl GitInfo {
-    pub fn new(dir: &Path) -> GitInfo {
+    pub fn new(config: &Config, dir: &Path) -> GitInfo {
         // See if this even begins to look like a git dir
-        if !dir.join(".git").exists() {
+        if config.ignore_git || !dir.join(".git").exists() {
             return GitInfo { inner: None }
         }
 

src/bootstrap/check.rs

@@ -164,7 +164,7 @@ impl Step for Cargotest {
         try_run(build, cmd.arg(&build.initial_cargo)
                           .arg(&out_dir)
                           .env("RUSTC", builder.rustc(compiler))
-                          .env("RUSTDOC", builder.rustdoc(compiler)));
+                          .env("RUSTDOC", builder.rustdoc(compiler.host)));
     }
 }
 
@@ -565,7 +565,7 @@ impl Step for Compiletest {
 
         // Avoid depending on rustdoc when we don't need it.
         if mode == "rustdoc" || mode == "run-make" {
-            cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
+            cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler.host));
         }
 
         cmd.arg("--src-base").arg(build.src.join("src/test").join(suite));
@@ -625,7 +625,7 @@ impl Step for Compiletest {
             cmd.arg("--system-llvm");
         }
 
-        cmd.args(&build.flags.cmd.test_args());
+        cmd.args(&build.config.cmd.test_args());
 
         if build.is_verbose() {
             cmd.arg("--verbose");
@@ -814,13 +814,13 @@ fn markdown_test(builder: &Builder, compiler: Compiler, markdown: &Path) {
     }
 
     println!("doc tests for: {}", markdown.display());
-    let mut cmd = builder.rustdoc_cmd(compiler);
+    let mut cmd = builder.rustdoc_cmd(compiler.host);
     build.add_rust_test_threads(&mut cmd);
     cmd.arg("--test");
     cmd.arg(markdown);
     cmd.env("RUSTC_BOOTSTRAP", "1");
 
-    let test_args = build.flags.cmd.test_args().join(" ");
+    let test_args = build.config.cmd.test_args().join(" ");
     cmd.arg("--test-args").arg(test_args);
 
     if build.config.quiet_tests {
@@ -1051,7 +1051,7 @@ impl Step for Crate {
         cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
         cargo.arg("--");
-        cargo.args(&build.flags.cmd.test_args());
+        cargo.args(&build.config.cmd.test_args());
 
         if build.config.quiet_tests {
             cargo.arg("--quiet");
@@ -1147,6 +1147,7 @@ pub struct Distcheck;
 
 impl Step for Distcheck {
     type Output = ();
+    const ONLY_BUILD: bool = true;
 
     fn should_run(run: ShouldRun) -> ShouldRun {
         run.path("distcheck")
@@ -1160,16 +1161,6 @@ impl Step for Distcheck {
     fn run(self, builder: &Builder) {
         let build = builder.build;
 
-        if *build.build != *"x86_64-unknown-linux-gnu" {
-            return
-        }
-        if !build.config.host.iter().any(|s| s == "x86_64-unknown-linux-gnu") {
-            return
-        }
-        if !build.config.target.iter().any(|s| s == "x86_64-unknown-linux-gnu") {
-            return
-        }
-
         println!("Distcheck");
         let dir = build.out.join("tmp").join("distcheck");
         let _ = fs::remove_dir_all(&dir);
@@ -1236,7 +1227,7 @@ impl Step for Bootstrap {
         if !build.fail_fast {
             cmd.arg("--no-fail-fast");
         }
-        cmd.arg("--").args(&build.flags.cmd.test_args());
+        cmd.arg("--").args(&build.config.cmd.test_args());
         try_run(build, &mut cmd);
     }
 

src/bootstrap/clean.rs

@@ -26,7 +26,7 @@ pub fn clean(build: &Build) {
     rm_rf(&build.out.join("tmp"));
     rm_rf(&build.out.join("dist"));
 
-    for host in build.config.host.iter() {
+    for host in &build.hosts {
         let entries = match build.out.join(host).read_dir() {
             Ok(iter) => iter,
             Err(_) => continue,

src/bootstrap/compile.rs

@@ -32,6 +32,7 @@ use serde_json;
 use util::{exe, libdir, is_dylib, copy};
 use {Build, Compiler, Mode};
 use native;
+use tool;
 
 use cache::{INTERNER, Interned};
 use builder::{Step, RunConfig, ShouldRun, Builder};
@@ -198,6 +199,12 @@ impl Step for StdLink {
             // for reason why the sanitizers are not built in stage0.
             copy_apple_sanitizer_dylibs(&build.native_dir(target), "osx", &libdir);
         }
+
+        builder.ensure(tool::CleanTools {
+            compiler: target_compiler,
+            target: target,
+            mode: Mode::Libstd,
+        });
     }
 }
 
@@ -389,6 +396,11 @@ impl Step for TestLink {
                 target);
         add_to_sysroot(&builder.sysroot_libdir(target_compiler, target),
                     &libtest_stamp(build, compiler, target));
+        builder.ensure(tool::CleanTools {
+            compiler: target_compiler,
+            target: target,
+            mode: Mode::Libtest,
+        });
     }
 }
 
@@ -567,6 +579,11 @@ impl Step for RustcLink {
                  target);
         add_to_sysroot(&builder.sysroot_libdir(target_compiler, target),
                        &librustc_stamp(build, compiler, target));
+        builder.ensure(tool::CleanTools {
+            compiler: target_compiler,
+            target: target,
+            mode: Mode::Librustc,
+        });
     }
 }
 
@@ -679,10 +696,10 @@ impl Step for Assemble {
         // link to these. (FIXME: Is that correct? It seems to be correct most
         // of the time but I think we do link to these for stage2/bin compilers
         // when not performing a full bootstrap).
-        if builder.build.flags.keep_stage.map_or(false, |s| target_compiler.stage <= s) {
+        if builder.build.config.keep_stage.map_or(false, |s| target_compiler.stage <= s) {
             builder.verbose("skipping compilation of compiler due to --keep-stage");
             let compiler = build_compiler;
-            for stage in 0..min(target_compiler.stage, builder.flags.keep_stage.unwrap()) {
+            for stage in 0..min(target_compiler.stage, builder.config.keep_stage.unwrap()) {
                 let target_compiler = builder.compiler(stage, target_compiler.host);
                 let target = target_compiler.host;
                 builder.ensure(StdLink { compiler, target_compiler, target });