metadata-link: add -Zmetadata-link flag

This adds the -Zmetadata-link flag, whose intent is to make the output
of `--emit metadata` equivalent to `--emit metadata,link`. The goal is
to allow pipelined builds with separate invocations of rustc. This is
desireable for two reasons:

1. The "artifact notification" system that rustc and cargo use to
  communicate is very cargo-specific - it doesn't fit well into other
  builds systems such as Buck or Bazel. In general its incompatible with
  any build system which only recognizes output artifacts once the
  compiler process has terminated. This is a particular problem when
  compilation is distributed.

2. The rmeta file could be cached independently from the rlib. For
  example, if one generates compilation-ready rmeta files as part
  of "check" build, then those can be directly used for a full
  compilation from cache, without having to regenerate them. This
  means the build turns into a highly parallelizable invocation of
  `rustc --emit link` commands using cached `rmeta` files as input.

Initially this flag `-Zmetadata-link`, but ultimately I'd expect to
stabilize it as `-Cmetadata-link` (or a better name).

Author: jsgf

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -45,7 +45,7 @@ pub fn crates_export_threshold(crate_types: &[CrateType]) -> SymbolExportLevel {
 fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<SymbolExportLevel> {
     assert_eq!(cnum, LOCAL_CRATE);
 
-    if !tcx.sess.opts.output_types.should_codegen() {
+    if !tcx.sess.opts.output_types.should_codegen() && !tcx.sess.opts.debugging_opts.metadata_link {
         return Default::default();
     }
 
@@ -164,7 +164,7 @@ fn exported_symbols_provider_local(
 ) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportLevel)] {
     assert_eq!(cnum, LOCAL_CRATE);
 
-    if !tcx.sess.opts.output_types.should_codegen() {
+    if !tcx.sess.opts.output_types.should_codegen() && !tcx.sess.opts.debugging_opts.metadata_link {
         return &[];
     }
 

compiler/rustc_interface/src/tests.rs

@@ -706,6 +706,7 @@ fn test_debugging_options_tracking_hash() {
     tracked!(instrument_mcount, true);
     tracked!(link_only, true);
     tracked!(merge_functions, Some(MergeFunctions::Disabled));
+    tracked!(metadata_link, true);
     tracked!(mir_emit_retag, true);
     tracked!(mir_opt_level, Some(4));
     tracked!(mutable_noalias, Some(true));

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -815,11 +815,14 @@ fn should_encode_stability(def_kind: DefKind) -> bool {
 ///
 /// Return a pair, resp. for CTFE and for LLVM.
 fn should_encode_mir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> (bool, bool) {
+    let always_encode_mir = tcx.sess.opts.debugging_opts.always_encode_mir;
+    let metadata_link = tcx.sess.opts.debugging_opts.metadata_link;
+
     match tcx.def_kind(def_id) {
         // Constructors
         DefKind::Ctor(_, _) => {
-            let mir_opt_base = tcx.sess.opts.output_types.should_codegen()
-                || tcx.sess.opts.debugging_opts.always_encode_mir;
+            let mir_opt_base =
+                tcx.sess.opts.output_types.should_codegen() || always_encode_mir || metadata_link;
             (true, mir_opt_base)
         }
         // Constants
@@ -831,19 +834,17 @@ fn should_encode_mir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> (bool, bool) {
             let generics = tcx.generics_of(def_id);
             let needs_inline = (generics.requires_monomorphization(tcx)
                 || tcx.codegen_fn_attrs(def_id).requests_inline())
-                && tcx.sess.opts.output_types.should_codegen();
+                && (tcx.sess.opts.output_types.should_codegen() || metadata_link);
             // Only check the presence of the `const` modifier.
             let is_const_fn = tcx.is_const_fn_raw(def_id.to_def_id());
-            let always_encode_mir = tcx.sess.opts.debugging_opts.always_encode_mir;
             (is_const_fn, needs_inline || always_encode_mir)
         }
         // Closures can't be const fn.
         DefKind::Closure => {
             let generics = tcx.generics_of(def_id);
             let needs_inline = (generics.requires_monomorphization(tcx)
                 || tcx.codegen_fn_attrs(def_id).requests_inline())
-                && tcx.sess.opts.output_types.should_codegen();
-            let always_encode_mir = tcx.sess.opts.debugging_opts.always_encode_mir;
+                && (tcx.sess.opts.output_types.should_codegen() || metadata_link);
             (false, needs_inline || always_encode_mir)
         }
         // Generators require optimized MIR to compute layout.
@@ -2020,7 +2021,7 @@ impl<'tcx, 'v> ItemLikeVisitor<'v> for ImplVisitor<'tcx> {
 /// Used to prefetch queries which will be needed later by metadata encoding.
 /// Only a subset of the queries are actually prefetched to keep this code smaller.
 fn prefetch_mir(tcx: TyCtxt<'_>) {
-    if !tcx.sess.opts.output_types.should_codegen() {
+    if !tcx.sess.opts.output_types.should_codegen() && !tcx.sess.opts.debugging_opts.metadata_link {
         // We won't emit MIR, so don't prefetch it.
         return;
     }

compiler/rustc_session/src/options.rs

@@ -1121,6 +1121,8 @@ options! {
         the same values as the target option of the same name"),
     meta_stats: bool = (false, parse_bool, [UNTRACKED],
         "gather metadata statistics (default: no)"),
+    metadata_link: bool = (false, parse_bool, [TRACKED],
+        "always emit metadata that's equivalent to metadata,link"),
     mir_emit_retag: bool = (false, parse_bool, [TRACKED],
         "emit Retagging MIR statements, interpreted e.g., by miri; implies -Zmir-opt-level=0 \
         (default: no)"),