Auto merge of #70642 - eddyb:remap-sysroot-src, r=Mark-Simulacrum

Translate the virtual `/rustc/$hash` prefix back to a real directory.

Closes #53486 and fixes #53081, by undoing the remapping to `/rustc/$hash` on the fly, when appropriate (e.g. our testsuites, or user crates that depend on `libstd`), but not during the Rust build itself (as that could leak the absolute build directory into the artifacts, breaking deterministic builds).

Tested locally by setting `remap-debuginfo = true` in `config.toml`, which without these changes, was causing 56 tests to fail (see #53081 (comment) for more details).

cc @Mark-Simulacrum @alexcrichton @ehuss

Author: bors

src/bootstrap/builder.rs

@@ -1022,8 +1022,13 @@ impl<'a> Builder<'a> {
             cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string());
         }
 
-        if let Some(map) = self.build.debuginfo_map(GitRepo::Rustc) {
+        if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) {
+            let map = format!("{}={}", self.build.src.display(), map_to);
             cargo.env("RUSTC_DEBUGINFO_MAP", map);
+
+            // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to,
+            // in order to opportunistically reverse it later.
+            cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to);
         }
 
         // Enable usage of unstable features

src/bootstrap/compile.rs

@@ -22,7 +22,7 @@ use serde::Deserialize;
 use crate::builder::Cargo;
 use crate::dist;
 use crate::native;
-use crate::util::{exe, is_dylib};
+use crate::util::{exe, is_dylib, symlink_dir};
 use crate::{Compiler, GitRepo, Mode};
 
 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
@@ -633,6 +633,30 @@ impl Step for Sysroot {
         };
         let _ = fs::remove_dir_all(&sysroot);
         t!(fs::create_dir_all(&sysroot));
+
+        // Symlink the source root into the same location inside the sysroot,
+        // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`),
+        // so that any tools relying on `rust-src` also work for local builds,
+        // and also for translating the virtual `/rustc/$hash` back to the real
+        // directory (for running tests with `rust.remap-debuginfo = true`).
+        let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src");
+        t!(fs::create_dir_all(&sysroot_lib_rustlib_src));
+        let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust");
+        if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) {
+            eprintln!(
+                "warning: creating symbolic link `{}` to `{}` failed with {}",
+                sysroot_lib_rustlib_src_rust.display(),
+                builder.src.display(),
+                e,
+            );
+            if builder.config.rust_remap_debuginfo {
+                eprintln!(
+                    "warning: some `src/test/ui` tests will fail when lacking `{}`",
+                    sysroot_lib_rustlib_src_rust.display(),
+                );
+            }
+        }
+
         INTERNER.intern_path(sysroot)
     }
 }

src/bootstrap/lib.rs

@@ -740,19 +740,18 @@ impl Build {
         self.config.jobs.unwrap_or_else(|| num_cpus::get() as u32)
     }
 
-    fn debuginfo_map(&self, which: GitRepo) -> Option<String> {
+    fn debuginfo_map_to(&self, which: GitRepo) -> Option<String> {
         if !self.config.rust_remap_debuginfo {
             return None;
         }
 
-        let path = match which {
+        match which {
             GitRepo::Rustc => {
                 let sha = self.rust_sha().unwrap_or(channel::CFG_RELEASE_NUM);
-                format!("/rustc/{}", sha)
+                Some(format!("/rustc/{}", sha))
             }
-            GitRepo::Llvm => String::from("/rustc/llvm"),
-        };
-        Some(format!("{}={}", self.src.display(), path))
+            GitRepo::Llvm => Some(String::from("/rustc/llvm")),
+        }
     }
 
     /// Returns the path to the C compiler for the target specified.
@@ -787,7 +786,8 @@ impl Build {
             base.push("-fno-omit-frame-pointer".into());
         }
 
-        if let Some(map) = self.debuginfo_map(which) {
+        if let Some(map_to) = self.debuginfo_map_to(which) {
+            let map = format!("{}={}", self.src.display(), map_to);
             let cc = self.cc(target);
             if cc.ends_with("clang") || cc.ends_with("gcc") {
                 base.push(format!("-fdebug-prefix-map={}", map));

src/etc/generate-deriving-span-tests.py

@@ -15,9 +15,6 @@
     os.path.join(os.path.dirname(__file__), '../test/ui/derives/'))
 
 TEMPLATE = """\
-// FIXME: missing sysroot spans (#53081)
-// ignore-i586-unknown-linux-gnu
-// ignore-i586-unknown-linux-musl
 // This file was auto-generated using 'src/etc/generate-deriving-span-tests.py'
 
 {error_deriving}

src/librustc_metadata/build.rs

@@ -1,4 +1,5 @@
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
     println!("cargo:rerun-if-env-changed=CFG_VERSION");
+    println!("cargo:rerun-if-env-changed=CFG_VIRTUAL_RUST_SOURCE_BASE_DIR");
 }

src/librustc_metadata/rmeta/decoder.rs

@@ -32,7 +32,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::util::common::record_time;
 use rustc_serialize::{opaque, Decodable, Decoder, SpecializedDecoder};
 use rustc_session::Session;
-use rustc_span::source_map::{self, respan, Spanned};
+use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{self, hygiene::MacroKind, BytePos, Pos, Span, DUMMY_SP};
 
@@ -41,6 +41,7 @@ use proc_macro::bridge::client::ProcMacro;
 use std::io;
 use std::mem;
 use std::num::NonZeroUsize;
+use std::path::Path;
 use std::u32;
 
 pub use cstore_impl::{provide, provide_extern};
@@ -427,7 +428,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
         // we can call `imported_source_files` for the proper crate, and binary search
         // through the returned slice using our span.
         let imported_source_files = if tag == TAG_VALID_SPAN_LOCAL {
-            self.cdata().imported_source_files(sess.source_map())
+            self.cdata().imported_source_files(sess)
         } else {
             // FIXME: We don't decode dependencies of proc-macros.
             // Remove this once #69976 is merged
@@ -457,7 +458,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
             self.last_source_file_index = 0;
 
             let foreign_data = self.cdata().cstore.get_crate_data(cnum);
-            foreign_data.imported_source_files(sess.source_map())
+            foreign_data.imported_source_files(sess)
         };
 
         let source_file = {
@@ -1460,10 +1461,45 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     ///
     /// Proc macro crates don't currently export spans, so this function does not have
     /// to work for them.
-    fn imported_source_files(
-        &self,
-        local_source_map: &source_map::SourceMap,
-    ) -> &'a [ImportedSourceFile] {
+    fn imported_source_files(&self, sess: &Session) -> &'a [ImportedSourceFile] {
+        // Translate the virtual `/rustc/$hash` prefix back to a real directory
+        // that should hold actual sources, where possible.
+        let virtual_rust_source_base_dir = option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR")
+            .map(Path::new)
+            .filter(|_| {
+                // Only spend time on further checks if we have what to translate *to*.
+                sess.real_rust_source_base_dir.is_some()
+            })
+            .filter(|virtual_dir| {
+                // Don't translate away `/rustc/$hash` if we're still remapping to it,
+                // since that means we're still building `std`/`rustc` that need it,
+                // and we don't want the real path to leak into codegen/debuginfo.
+                !sess.opts.remap_path_prefix.iter().any(|(_from, to)| to == virtual_dir)
+            });
+        let try_to_translate_virtual_to_real = |name: &mut rustc_span::FileName| {
+            debug!(
+                "try_to_translate_virtual_to_real(name={:?}): \
+                 virtual_rust_source_base_dir={:?}, real_rust_source_base_dir={:?}",
+                name, virtual_rust_source_base_dir, sess.real_rust_source_base_dir,
+            );
+
+            if let Some(virtual_dir) = virtual_rust_source_base_dir {
+                if let Some(real_dir) = &sess.real_rust_source_base_dir {
+                    if let rustc_span::FileName::Real(path) = name {
+                        if let Ok(rest) = path.strip_prefix(virtual_dir) {
+                            let new_path = real_dir.join(rest);
+                            debug!(
+                                "try_to_translate_virtual_to_real: `{}` -> `{}`",
+                                path.display(),
+                                new_path.display(),
+                            );
+                            *path = new_path;
+                        }
+                    }
+                }
+            }
+        };
+
         self.cdata.source_map_import_info.init_locking(|| {
             let external_source_map = self.root.source_map.decode(self);
 
@@ -1472,7 +1508,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                     // We can't reuse an existing SourceFile, so allocate a new one
                     // containing the information we need.
                     let rustc_span::SourceFile {
-                        name,
+                        mut name,
                         name_was_remapped,
                         src_hash,
                         start_pos,
@@ -1485,6 +1521,13 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                         ..
                     } = source_file_to_import;
 
+                    // If this file's path has been remapped to `/rustc/$hash`,
+                    // we might be able to reverse that (also see comments above,
+                    // on `try_to_translate_virtual_to_real`).
+                    // FIXME(eddyb) we could check `name_was_remapped` here,
+                    // but in practice it seems to be always `false`.
+                    try_to_translate_virtual_to_real(&mut name);
+
                     let source_length = (end_pos - start_pos).to_usize();
 
                     // Translate line-start positions and multibyte character
@@ -1505,7 +1548,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                         np.pos = np.pos - start_pos;
                     }
 
-                    let local_version = local_source_map.new_imported_source_file(
+                    let local_version = sess.source_map().new_imported_source_file(
                         name,
                         name_was_remapped,
                         src_hash,

src/librustc_session/session.rs

@@ -140,6 +140,15 @@ pub struct Session {
     /// Options range from returning the error without a backtrace to returning an error
     /// and immediately printing the backtrace to stderr.
     pub ctfe_backtrace: Lock<CtfeBacktrace>,
+
+    /// Base directory containing the `src/` for the Rust standard library, and
+    /// potentially `rustc` as well, if we can can find it. Right now it's always
+    /// `$sysroot/lib/rustlib/src/rust` (i.e. the `rustup` `rust-src` component).
+    ///
+    /// This directory is what the virtual `/rustc/$hash` is translated back to,
+    /// if Rust was built with path remapping to `/rustc/$hash` enabled
+    /// (the `rust.remap-debuginfo` option in `config.toml`).
+    pub real_rust_source_base_dir: Option<PathBuf>,
 }
 
 pub struct PerfStats {
@@ -1056,6 +1065,26 @@ fn build_session_(
         _ => CtfeBacktrace::Disabled,
     });
 
+    // Try to find a directory containing the Rust `src`, for more details see
+    // the doc comment on the `real_rust_source_base_dir` field.
+    let real_rust_source_base_dir = {
+        // This is the location used by the `rust-src` `rustup` component.
+        let mut candidate = sysroot.join("lib/rustlib/src/rust");
+        if let Ok(metadata) = candidate.symlink_metadata() {
+            // Replace the symlink rustbuild creates, with its destination.
+            // We could try to use `fs::canonicalize` instead, but that might
+            // produce unnecessarily verbose path.
+            if metadata.file_type().is_symlink() {
+                if let Ok(symlink_dest) = std::fs::read_link(&candidate) {
+                    candidate = symlink_dest;
+                }
+            }
+        }
+
+        // Only use this directory if it has a file we can expect to always find.
+        if candidate.join("src/libstd/lib.rs").is_file() { Some(candidate) } else { None }
+    };
+
     let sess = Session {
         target: target_cfg,
         host,
@@ -1094,6 +1123,7 @@ fn build_session_(
         confused_type_with_std_module: Lock::new(Default::default()),
         system_library_path: OneThread::new(RefCell::new(Default::default())),
         ctfe_backtrace,
+        real_rust_source_base_dir,
     };
 
     validate_commandline_args_with_session_available(&sess);

src/test/ui/async-await/issues/issue-62009-1.rs

@@ -1,8 +1,4 @@
 // edition:2018
-// FIXME: missing sysroot spans (#53081)
-// ignore-i586-unknown-linux-gnu
-// ignore-i586-unknown-linux-musl
-// ignore-i686-unknown-linux-musl
 
 async fn print_dur() {}
 

src/test/ui/async-await/issues/issue-62009-1.stderr

@@ -1,13 +1,13 @@
 error[E0728]: `await` is only allowed inside `async` functions and blocks
-  --> $DIR/issue-62009-1.rs:10:5
+  --> $DIR/issue-62009-1.rs:6:5
    |
 LL | fn main() {
    |    ---- this is not `async`
 LL |     async { let (); }.await;
    |     ^^^^^^^^^^^^^^^^^^^^^^^ only allowed inside `async` functions and blocks
 
 error[E0728]: `await` is only allowed inside `async` functions and blocks
-  --> $DIR/issue-62009-1.rs:12:5
+  --> $DIR/issue-62009-1.rs:8:5
    |
 LL |   fn main() {
    |      ---- this is not `async`
@@ -19,19 +19,19 @@ LL | |     }.await;
    | |___________^ only allowed inside `async` functions and blocks
 
 error[E0728]: `await` is only allowed inside `async` functions and blocks
-  --> $DIR/issue-62009-1.rs:16:5
+  --> $DIR/issue-62009-1.rs:12:5
    |
 LL | fn main() {
    |    ---- this is not `async`
 ...
 LL |     (|_| 2333).await;
    |     ^^^^^^^^^^^^^^^^ only allowed inside `async` functions and blocks
 
-error[E0277]: the trait bound `[closure@$DIR/issue-62009-1.rs:16:5: 16:15]: std::future::Future` is not satisfied
-  --> $DIR/issue-62009-1.rs:16:5
+error[E0277]: the trait bound `[closure@$DIR/issue-62009-1.rs:12:5: 12:15]: std::future::Future` is not satisfied
+  --> $DIR/issue-62009-1.rs:12:5
    |
 LL |     (|_| 2333).await;
-   |     ^^^^^^^^^^^^^^^^ the trait `std::future::Future` is not implemented for `[closure@$DIR/issue-62009-1.rs:16:5: 16:15]`
+   |     ^^^^^^^^^^^^^^^^ the trait `std::future::Future` is not implemented for `[closure@$DIR/issue-62009-1.rs:12:5: 12:15]`
    | 
   ::: $SRC_DIR/libcore/future/mod.rs:LL:COL
    |

src/test/ui/closures/closure-move-sync.rs

@@ -1,7 +1,3 @@
-// FIXME: missing sysroot spans (#53081)
-// ignore-i586-unknown-linux-gnu
-// ignore-i586-unknown-linux-musl
-// ignore-i686-unknown-linux-musl
 use std::thread;
 use std::sync::mpsc::channel;
 

src/test/ui/closures/closure-move-sync.stderr

@@ -1,5 +1,5 @@
 error[E0277]: `std::sync::mpsc::Receiver<()>` cannot be shared between threads safely
-  --> $DIR/closure-move-sync.rs:10:13
+  --> $DIR/closure-move-sync.rs:6:13
    |
 LL |     let t = thread::spawn(|| {
    |             ^^^^^^^^^^^^^ `std::sync::mpsc::Receiver<()>` cannot be shared between threads safely
@@ -11,10 +11,10 @@ LL |     F: Send + 'static,
    |
    = help: the trait `std::marker::Sync` is not implemented for `std::sync::mpsc::Receiver<()>`
    = note: required because of the requirements on the impl of `std::marker::Send` for `&std::sync::mpsc::Receiver<()>`
-   = note: required because it appears within the type `[closure@$DIR/closure-move-sync.rs:10:27: 13:6 recv:&std::sync::mpsc::Receiver<()>]`
+   = note: required because it appears within the type `[closure@$DIR/closure-move-sync.rs:6:27: 9:6 recv:&std::sync::mpsc::Receiver<()>]`
 
 error[E0277]: `std::sync::mpsc::Sender<()>` cannot be shared between threads safely
-  --> $DIR/closure-move-sync.rs:22:5
+  --> $DIR/closure-move-sync.rs:18:5
    |
 LL |     thread::spawn(|| tx.send(()).unwrap());
    |     ^^^^^^^^^^^^^ `std::sync::mpsc::Sender<()>` cannot be shared between threads safely
@@ -26,7 +26,7 @@ LL |     F: Send + 'static,
    |
    = help: the trait `std::marker::Sync` is not implemented for `std::sync::mpsc::Sender<()>`
    = note: required because of the requirements on the impl of `std::marker::Send` for `&std::sync::mpsc::Sender<()>`
-   = note: required because it appears within the type `[closure@$DIR/closure-move-sync.rs:22:19: 22:42 tx:&std::sync::mpsc::Sender<()>]`
+   = note: required because it appears within the type `[closure@$DIR/closure-move-sync.rs:18:19: 18:42 tx:&std::sync::mpsc::Sender<()>]`
 
 error: aborting due to 2 previous errors
 

src/test/ui/consts/const-size_of-cycle.rs

@@ -1,7 +1,3 @@
-// FIXME: missing sysroot spans (#53081)
-// ignore-i586-unknown-linux-gnu
-// ignore-i586-unknown-linux-musl
-// ignore-i686-unknown-linux-musl
 // error-pattern: cycle detected
 
 struct Foo {

src/test/ui/consts/const-size_of-cycle.stderr

@@ -1,16 +1,16 @@
 error[E0391]: cycle detected when const-evaluating + checking `Foo::bytes::{{constant}}#0`
-  --> $DIR/const-size_of-cycle.rs:8:17
+  --> $DIR/const-size_of-cycle.rs:4:17
    |
 LL |     bytes: [u8; std::mem::size_of::<Foo>()]
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
 note: ...which requires const-evaluating + checking `Foo::bytes::{{constant}}#0`...
-  --> $DIR/const-size_of-cycle.rs:8:17
+  --> $DIR/const-size_of-cycle.rs:4:17
    |
 LL |     bytes: [u8; std::mem::size_of::<Foo>()]
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 note: ...which requires const-evaluating `Foo::bytes::{{constant}}#0`...
-  --> $DIR/const-size_of-cycle.rs:8:17
+  --> $DIR/const-size_of-cycle.rs:4:17
    |
 LL |     bytes: [u8; std::mem::size_of::<Foo>()]
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -28,7 +28,7 @@ LL |     pub fn size_of<T>() -> usize;
    = note: ...which requires normalizing `[u8; _]`...
    = note: ...which again requires const-evaluating + checking `Foo::bytes::{{constant}}#0`, completing the cycle
 note: cycle used when processing `Foo`
-  --> $DIR/const-size_of-cycle.rs:7:1
+  --> $DIR/const-size_of-cycle.rs:3:1
    |
 LL | struct Foo {
    | ^^^^^^^^^^