Implement the translation item collector.

The purpose of the translation item collector is to find all monomorphic instances of functions, methods and statics that need to be translated into LLVM IR in order to compile the current crate.
So far these instances have been discovered lazily during the trans path. For incremental compilation we want to know the set of these instances in advance, and that is what the trans::collect module provides.
In the future, incremental and regular translation will be driven by the collector implemented here.

Author: michaelwoerister

configure

@@ -1391,6 +1391,7 @@ do
     make_dir $h/test/debuginfo-gdb
     make_dir $h/test/debuginfo-lldb
     make_dir $h/test/codegen
+    make_dir $h/test/codegen-units
     make_dir $h/test/rustdoc
 done
 

mk/tests.mk

@@ -321,6 +321,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
 	check-stage$(1)-T-$(2)-H-$(3)-debuginfo-gdb-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-debuginfo-lldb-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-codegen-exec \
+	check-stage$(1)-T-$(2)-H-$(3)-codegen-units-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-doc-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-pretty-exec
 
@@ -484,6 +485,7 @@ DEBUGINFO_GDB_RS := $(wildcard $(S)src/test/debuginfo/*.rs)
 DEBUGINFO_LLDB_RS := $(wildcard $(S)src/test/debuginfo/*.rs)
 CODEGEN_RS := $(wildcard $(S)src/test/codegen/*.rs)
 CODEGEN_CC := $(wildcard $(S)src/test/codegen/*.cc)
+CODEGEN_UNITS_RS := $(wildcard $(S)src/test/codegen-units/*.rs)
 RUSTDOCCK_RS := $(wildcard $(S)src/test/rustdoc/*.rs)
 
 # perf tests are the same as bench tests only they run under
@@ -504,6 +506,7 @@ PRETTY_TESTS := $(PRETTY_RS)
 DEBUGINFO_GDB_TESTS := $(DEBUGINFO_GDB_RS)
 DEBUGINFO_LLDB_TESTS := $(DEBUGINFO_LLDB_RS)
 CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
+CODEGEN_UNITS_TESTS := $(CODEGEN_UNITS_RS)
 RUSTDOCCK_TESTS := $(RUSTDOCCK_RS)
 
 CTEST_SRC_BASE_rpass = run-pass
@@ -571,6 +574,11 @@ CTEST_BUILD_BASE_codegen = codegen
 CTEST_MODE_codegen = codegen
 CTEST_RUNTOOL_codegen = $(CTEST_RUNTOOL)
 
+CTEST_SRC_BASE_codegen-units = codegen-units
+CTEST_BUILD_BASE_codegen-units = codegen-units
+CTEST_MODE_codegen-units = codegen-units
+CTEST_RUNTOOL_codegen-units = $(CTEST_RUNTOOL)
+
 CTEST_SRC_BASE_rustdocck = rustdoc
 CTEST_BUILD_BASE_rustdocck = rustdoc
 CTEST_MODE_rustdocck = rustdoc
@@ -695,6 +703,7 @@ CTEST_DEPS_debuginfo-lldb_$(1)-T-$(2)-H-$(3) = $$(DEBUGINFO_LLDB_TESTS) \
                                                $(S)src/etc/lldb_batchmode.py \
                                                $(S)src/etc/lldb_rust_formatters.py
 CTEST_DEPS_codegen_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_TESTS)
+CTEST_DEPS_codegen-units_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_UNITS_TESTS)
 CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
         $$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
 	$(S)src/etc/htmldocck.py
@@ -761,7 +770,7 @@ endif
 endef
 
 CTEST_NAMES = rpass rpass-valgrind rpass-full rfail-full cfail-full rfail cfail pfail \
-	bench perf debuginfo-gdb debuginfo-lldb codegen rustdocck
+	bench perf debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck
 
 $(foreach host,$(CFG_HOST), \
  $(eval $(foreach target,$(CFG_TARGET), \
@@ -940,6 +949,7 @@ TEST_GROUPS = \
 	debuginfo-gdb \
 	debuginfo-lldb \
 	codegen \
+	codegen-units \
 	doc \
 	$(foreach docname,$(DOC_NAMES),doc-$(docname)) \
 	pretty \

src/compiletest/common.rs

@@ -25,6 +25,7 @@ pub enum Mode {
     DebugInfoLldb,
     Codegen,
     Rustdoc,
+    CodegenUnits
 }
 
 impl FromStr for Mode {
@@ -41,6 +42,7 @@ impl FromStr for Mode {
           "debuginfo-gdb" => Ok(DebugInfoGdb),
           "codegen" => Ok(Codegen),
           "rustdoc" => Ok(Rustdoc),
+          "codegen-units" => Ok(CodegenUnits),
           _ => Err(()),
         }
     }
@@ -59,6 +61,7 @@ impl fmt::Display for Mode {
             DebugInfoLldb => "debuginfo-lldb",
             Codegen => "codegen",
             Rustdoc => "rustdoc",
+            CodegenUnits => "codegen-units",
         }, f)
     }
 }

src/compiletest/runtest.rs

@@ -10,14 +10,15 @@
 
 use common::Config;
 use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
-use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc};
+use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
 use errors;
 use header::TestProps;
 use header;
 use procsrv;
 use util::logv;
 
 use std::env;
+use std::collections::HashSet;
 use std::fmt;
 use std::fs::{self, File};
 use std::io::BufReader;
@@ -56,6 +57,7 @@ pub fn run(config: Config, testfile: &Path) {
         DebugInfoLldb => run_debuginfo_lldb_test(&config, &props, &testfile),
         Codegen => run_codegen_test(&config, &props, &testfile),
         Rustdoc => run_rustdoc_test(&config, &props, &testfile),
+        CodegenUnits => run_codegen_units_test(&config, &props, &testfile),
     }
 }
 
@@ -1747,3 +1749,44 @@ fn run_rustdoc_test(config: &Config, props: &TestProps, testfile: &Path) {
         fatal_proc_rec("htmldocck failed!", &res);
     }
 }
+
+fn run_codegen_units_test(config: &Config, props: &TestProps, testfile: &Path) {
+    let proc_res = compile_test(config, props, testfile);
+
+    if !proc_res.status.success() {
+        fatal_proc_rec("compilation failed!", &proc_res);
+    }
+
+    check_no_compiler_crash(&proc_res);
+
+    let prefix = "TRANS_ITEM ";
+
+    let actual: HashSet<String> = proc_res
+        .stdout
+        .lines()
+        .filter(|line| line.starts_with(prefix))
+        .map(|s| (&s[prefix.len()..]).to_string())
+        .collect();
+
+    let expected: HashSet<String> = errors::load_errors(testfile)
+        .iter()
+        .map(|e| e.msg.trim().to_string())
+        .collect();
+
+    if actual != expected {
+        let mut missing: Vec<_> = expected.difference(&actual).collect();
+        missing.sort();
+
+        let mut too_much: Vec<_> = actual.difference(&expected).collect();
+        too_much.sort();
+
+        println!("Expected and actual sets of codegen-items differ.\n\
+                  These items should have been contained but were not:\n\n\
+                  {}\n\n\
+                  These items were contained but should not have been:\n\n\
+                  {}\n\n",
+            missing.iter().fold("".to_string(), |s1, s2| s1 + "\n" + s2),
+            too_much.iter().fold("".to_string(), |s1, s2| s1 + "\n" + s2));
+        panic!();
+    }
+}

src/librustc/front/map/definitions.rs

@@ -196,33 +196,33 @@ impl DefPathData {
 
             PositionalField |
             Field(hir::StructFieldKind::UnnamedField(_)) => {
-                InternedString::new("<field>")
+                InternedString::new("{{field}}")
             }
 
             // note that this does not show up in user printouts
             CrateRoot => {
-                InternedString::new("<root>")
+                InternedString::new("{{root}}")
             }
 
             // note that this does not show up in user printouts
             InlinedRoot(_) => {
-                InternedString::new("<inlined-root>")
+                InternedString::new("{{inlined-root}}")
             }
 
             Misc => {
-                InternedString::new("?")
+                InternedString::new("{{?}}")
             }
 
             ClosureExpr => {
-                InternedString::new("<closure>")
+                InternedString::new("{{closure}}")
             }
 
             StructCtor => {
-                InternedString::new("<constructor>")
+                InternedString::new("{{constructor}}")
             }
 
             Initializer => {
-                InternedString::new("<initializer>")
+                InternedString::new("{{initializer}}")
             }
         }
     }

src/librustc/middle/ty/context.rs

@@ -408,6 +408,9 @@ pub struct ctxt<'tcx> {
     /// fragmented data to the set of unfragmented pieces that
     /// constitute it.
     pub fragment_infos: RefCell<DefIdMap<Vec<ty::FragmentInfo>>>,
+
+    /// Maps inlined closures from their original DefId to their local NodeId
+    pub inlined_closures: RefCell<DepTrackingMap<maps::InlinedClosures<'tcx>>>,
 }
 
 impl<'tcx> ctxt<'tcx> {
@@ -563,6 +566,7 @@ impl<'tcx> ctxt<'tcx> {
             custom_coerce_unsized_kinds: RefCell::new(DefIdMap()),
             cast_kinds: RefCell::new(NodeMap()),
             fragment_infos: RefCell::new(DefIdMap()),
+            inlined_closures: RefCell::new(DepTrackingMap::new(dep_graph.clone())),
        }, f)
     }
 }

src/librustc/middle/ty/maps.rs

@@ -13,7 +13,7 @@ use middle::def_id::DefId;
 use middle::ty;
 use std::marker::PhantomData;
 use std::rc::Rc;
-use syntax::attr;
+use syntax::{attr, ast};
 
 macro_rules! dep_map_ty {
     ($ty_name:ident : $node_name:ident ($key:ty) -> $value:ty) => {
@@ -42,3 +42,4 @@ dep_map_ty! { InherentImpls: InherentImpls(DefId) -> Rc<Vec<DefId>> }
 dep_map_ty! { ImplItems: ImplItems(DefId) -> Vec<ty::ImplOrTraitItemId> }
 dep_map_ty! { TraitItems: TraitItems(DefId) -> Rc<Vec<ty::ImplOrTraitItem<'tcx>>> }
 dep_map_ty! { ReprHints: ReprHints(DefId) -> Rc<Vec<attr::ReprAttr>> }
+dep_map_ty! { InlinedClosures: Hir(DefId) -> ast::NodeId }

src/librustc/mir/repr.rs

@@ -720,7 +720,7 @@ pub enum Rvalue<'tcx> {
     InlineAsm(InlineAsm),
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, RustcEncodable, RustcDecodable)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, RustcEncodable, RustcDecodable)]
 pub enum CastKind {
     Misc,
 

src/librustc/session/config.rs

@@ -634,6 +634,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
           "keep the AST after lowering it to HIR"),
     show_span: Option<String> = (None, parse_opt_string,
           "show spans for compiler debugging (expr|pat|ty)"),
+    print_trans_items: Option<String> = (None, parse_opt_string,
+          "print the result of the translation item collection pass"),
 }
 
 pub fn default_lib_output() -> CrateType {

src/librustc_metadata/astencode.rs

@@ -14,10 +14,12 @@
 
 use rustc::front::map as ast_map;
 use rustc::session::Session;
+use rustc::util::common::MemoizationMap;
 
 use rustc_front::hir;
 use rustc_front::fold;
 use rustc_front::fold::Folder;
+use rustc_front::intravisit;
 
 use common as c;
 use cstore;
@@ -177,6 +179,8 @@ pub fn decode_inlined_item<'tcx>(cdata: &cstore::crate_metadata,
           }
           _ => { }
         }
+        register_inlined_closures(dcx.tcx, ii);
+
         Ok(ii)
       }
     }
@@ -1274,6 +1278,41 @@ fn inlined_item_id_range(v: &InlinedItem) -> ast_util::IdRange {
     visitor.result()
 }
 
+fn register_inlined_closures<'tcx, 'ii>(tcx: &ty::ctxt<'tcx>,
+                                        ii: &'ii InlinedItem)
+    where 'tcx: 'ii {
+
+    let mut visitor = ClosureVisitor { tcx: tcx };
+    ii.visit(&mut visitor);
+
+    struct ClosureVisitor<'a, 'tcx>
+        where 'tcx: 'a {
+        tcx: &'a ty::ctxt<'tcx>
+    }
+
+    impl<'a, 'tcx, 'v> intravisit::Visitor<'v> for ClosureVisitor<'a, 'tcx>
+        where 'tcx: 'a,
+              'tcx: 'v {
+
+        fn visit_fn(&mut self,
+                    fk: intravisit::FnKind<'v>,
+                    fd: &'v hir::FnDecl,
+                    b: &'v hir::Block,
+                    s: codemap::Span,
+                    id: ast::NodeId) {
+
+            if let intravisit::FnKind::Closure = fk {
+                let ty = self.tcx.node_id_to_type(id);
+                if let ty::TyClosure(def_id, _) = ty.sty {
+                    self.tcx.inlined_closures.memoize(def_id, || id );
+                }
+            }
+
+            intravisit::walk_fn(self, fk, fd, b, s)
+        }
+    }
+}
+
 // ______________________________________________________________________
 // Testing of astencode_gen
 

src/librustc_metadata/decoder.rs

@@ -834,6 +834,8 @@ pub fn maybe_get_item_mir<'tcx>(cdata: Cmd,
             })
         }).unwrap();
 
+        assert!(decoder.position() == mir_doc.end);
+
         let mut def_id_and_span_translator = MirDefIdAndSpanTranslator {
             crate_metadata: cdata,
             codemap: tcx.sess.codemap(),