From 16acdbd075486dae3fffd7cd4131a4c748f39978 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Wed, 12 Nov 2014 21:22:52 -0500 Subject: [PATCH] replace the system allocator in executables This adds support for replacing the system allocator with jemalloc by overriding the weak symbols on Linux (including Android). It is disabled by default with #![no_std] and can be toggled via a compiler switch. It will be possible to extend this to other platforms in the future. This results in a performance improvement for memory allocation in C along with reduced fragmentation. For example, the time spent on LLVM passes in the Rust compiler on Linux is cut by 10% and peak memory usage is reduced by 15%. Closes #18896 --- mk/clean.mk | 1 + mk/main.mk | 3 +- mk/rt.mk | 3 +- mk/target.mk | 6 ++ src/librustc/session/config.rs | 9 ++ src/librustc/session/mod.rs | 2 + src/librustc_back/target/linux_base.rs | 1 + src/librustc_back/target/mod.rs | 3 + src/librustc_trans/back/link.rs | 13 +++ src/librustc_trans/driver/driver.rs | 1 + src/libsyntax/std_inject.rs | 2 +- src/rt/rust_malloc.c | 111 +++++++++++++++++++++++++ 12 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 src/rt/rust_malloc.c diff --git a/mk/clean.mk b/mk/clean.mk index 97c823c9e2de9..d7cf184168cc2 100644 --- a/mk/clean.mk +++ b/mk/clean.mk @@ -101,6 +101,7 @@ clean$(1)_T_$(2)_H_$(3): \ $$(foreach tool,$$(TOOLS),clean$(1)_T_$(2)_H_$(3)-tool-$$(tool)) $$(Q)rm -f $$(TLIB$(1)_T_$(2)_H_$(3))/libmorestack.a $$(Q)rm -f $$(TLIB$(1)_T_$(2)_H_$(3))/libcompiler-rt.a + $$(Q)rm -f $$(TLIB$(1)_T_$(2)_H_$(3))/librust_malloc.a $(Q)rm -f $$(TLIB$(1)_T_$(2)_H_$(3))/librun_pass_stage* # For unix $(Q)rm -f $$(TLIB$(1)_T_$(2)_H_$(3))/run_pass_stage* # For windows diff --git a/mk/main.mk b/mk/main.mk index 3df4d3bfa5ec6..9402abb468bdb 100644 --- a/mk/main.mk +++ b/mk/main.mk @@ -352,7 +352,8 @@ endif TSREQ$(1)_T_$(2)_H_$(3) = \ $$(HSREQ$(1)_H_$(3)) \ $$(TLIB$(1)_T_$(2)_H_$(3))/libmorestack.a \ - $$(TLIB$(1)_T_$(2)_H_$(3))/libcompiler-rt.a + $$(TLIB$(1)_T_$(2)_H_$(3))/libcompiler-rt.a \ + $$(TLIB$(1)_T_$(2)_H_$(3))/librust_malloc.a # Prerequisites for a working stageN compiler and libraries, for a specific # target diff --git a/mk/rt.mk b/mk/rt.mk index a7d6a6e825fbc..604539d3f314e 100644 --- a/mk/rt.mk +++ b/mk/rt.mk @@ -36,7 +36,7 @@ # target. ################################################################################ NATIVE_LIBS := rust_builtin hoedown morestack miniz context_switch \ - rustrt_native rust_test_helpers + rustrt_native rust_test_helpers rust_malloc # $(1) is the target triple define NATIVE_LIBRARIES @@ -58,6 +58,7 @@ NATIVE_DEPS_rustrt_native_$(1) := \ arch/$$(HOST_$(1))/record_sp.S NATIVE_DEPS_rust_test_helpers_$(1) := rust_test_helpers.c NATIVE_DEPS_morestack_$(1) := arch/$$(HOST_$(1))/morestack.S +NATIVE_DEPS_rust_malloc_$(1) := rust_malloc.c NATIVE_DEPS_context_switch_$(1) := \ arch/$$(HOST_$(1))/_context.S diff --git a/mk/target.mk b/mk/target.mk index ed7d8bb497d28..aed539474c6be 100644 --- a/mk/target.mk +++ b/mk/target.mk @@ -153,6 +153,12 @@ $$(TLIB$(1)_T_$(2)_H_$(3))/libmorestack.a: \ | $$(TLIB$(1)_T_$(2)_H_$(3))/ $$(SNAPSHOT_RUSTC_POST_CLEANUP) @$$(call E, cp: $$@) $$(Q)cp $$< $$@ + +$$(TLIB$(1)_T_$(2)_H_$(3))/librust_malloc.a: \ + $$(RT_OUTPUT_DIR_$(2))/$$(call CFG_STATIC_LIB_NAME_$(2),rust_malloc) \ + | $$(TLIB$(1)_T_$(2)_H_$(3))/ $$(SNAPSHOT_RUSTC_POST_CLEANUP) + @$$(call E, cp: $$@) + $$(Q)cp $$< $$@ endef $(foreach source,$(CFG_HOST), \ diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index e10a1a4342c45..adf04669cdd80 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -318,6 +318,13 @@ macro_rules! cgoptions( } )* + fn parse_opt_toggle(slot: &mut Option, v: Option<&str>) -> bool { + match v.and_then(from_str) { + Some(b) => { *slot = Some(b); true }, + None => false + } + } + fn parse_bool(slot: &mut bool, v: Option<&str>) -> bool { match v { Some(..) => false, @@ -447,6 +454,8 @@ cgoptions!( "print remarks for these optimization passes (space separated, or \"all\")"), no_stack_check: bool = (false, parse_bool, "disable checks for stack exhaustion (a memory-safety hazard!)"), + replace_allocator: Option = (None, parse_opt_toggle, + "attempt to replace the system allocator with jemalloc"), ) pub fn build_codegen_options(matches: &getopts::Matches) -> CodegenOptions diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index 72a9f23aa1f94..c5f9f9bfd9a60 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -50,6 +50,7 @@ pub struct Session { pub crate_types: RefCell>, pub crate_metadata: RefCell>, pub features: RefCell, + pub use_std: Cell, /// The maximum recursion limit for potentially infinitely recursive /// operations such as auto-dereference and monomorphization. @@ -252,6 +253,7 @@ pub fn build_session_(sopts: config::Options, crate_types: RefCell::new(Vec::new()), crate_metadata: RefCell::new(Vec::new()), features: RefCell::new(feature_gate::Features::new()), + use_std: Cell::new(false), recursion_limit: Cell::new(64), }; diff --git a/src/librustc_back/target/linux_base.rs b/src/librustc_back/target/linux_base.rs index d267bc77e4975..104cc6810b3a7 100644 --- a/src/librustc_back/target/linux_base.rs +++ b/src/librustc_back/target/linux_base.rs @@ -26,6 +26,7 @@ pub fn opts() -> TargetOptions { "-Wl,--as-needed".to_string(), ), position_independent_executables: true, + weak_malloc: true, .. Default::default() } } diff --git a/src/librustc_back/target/mod.rs b/src/librustc_back/target/mod.rs index d7b4285cdb0ab..9d5caf29e2ab6 100644 --- a/src/librustc_back/target/mod.rs +++ b/src/librustc_back/target/mod.rs @@ -160,6 +160,8 @@ pub struct TargetOptions { /// advantage of ASLR, as otherwise the functions in the executable are not randomized and can /// be used during an exploit of a vulnerability in any code. pub position_independent_executables: bool, + /// The platform allocator can be replaced via weak symbols + pub weak_malloc: bool } impl Default for TargetOptions { @@ -191,6 +193,7 @@ impl Default for TargetOptions { has_rpath: false, no_compiler_rt: false, position_independent_executables: false, + weak_malloc: false } } } diff --git a/src/librustc_trans/back/link.rs b/src/librustc_trans/back/link.rs index d5d488e8965a3..cc7670ecd2394 100644 --- a/src/librustc_trans/back/link.rs +++ b/src/librustc_trans/back/link.rs @@ -885,6 +885,19 @@ fn link_args(cmd: &mut Command, } } + if !dylib && t.options.weak_malloc && + sess.opts.cg.replace_allocator.unwrap_or(sess.use_std.get()) { + if t.options.is_like_osx { + let rust_malloc = lib_path.join("librust_malloc.a"); + + let mut v = b"-Wl,-force_load,".to_vec(); + v.push_all(rust_malloc.as_vec()); + cmd.arg(v.as_slice()); + } else { + cmd.args(["-Wl,--whole-archive", "-lrust_malloc", "-Wl,--no-whole-archive"]); + } + } + // When linking a dynamic library, we put the metadata into a section of the // executable. This metadata is in a separate object file from the main // object file, so we link that in here. diff --git a/src/librustc_trans/driver/driver.rs b/src/librustc_trans/driver/driver.rs index a0e2bf07b830f..c413ba1c8bb10 100644 --- a/src/librustc_trans/driver/driver.rs +++ b/src/librustc_trans/driver/driver.rs @@ -182,6 +182,7 @@ pub fn phase_2_configure_and_expand(sess: &Session, collect_crate_types(sess, krate.attrs.as_slice()); *sess.crate_metadata.borrow_mut() = collect_crate_metadata(sess, krate.attrs.as_slice()); + sess.use_std.set(syntax::std_inject::use_std(&krate)); time(time_passes, "gated feature checking", (), |_| { let (features, unknown_features) = diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs index 6a4ab365a50b2..f96fa6ea419bf 100644 --- a/src/libsyntax/std_inject.rs +++ b/src/libsyntax/std_inject.rs @@ -39,7 +39,7 @@ pub fn maybe_inject_prelude(krate: ast::Crate) -> ast::Crate { } } -fn use_std(krate: &ast::Crate) -> bool { +pub fn use_std(krate: &ast::Crate) -> bool { !attr::contains_name(krate.attrs.as_slice(), "no_std") } diff --git a/src/rt/rust_malloc.c b/src/rt/rust_malloc.c new file mode 100644 index 0000000000000..44298b39ab236 --- /dev/null +++ b/src/rt/rust_malloc.c @@ -0,0 +1,111 @@ +#include + +void *je_malloc(size_t size); +void *je_calloc(size_t num, size_t size); +int je_posix_memalign(void **memptr, size_t alignment, size_t size); +void *je_aligned_alloc(size_t alignment, size_t size); +void *je_realloc(void *ptr, size_t size); +void je_free(void *ptr); + +void *je_mallocx(size_t size, int flags); +void *je_rallocx(void *ptr, size_t size, int flags); +size_t je_xallocx(void *ptr, size_t size, size_t extra, int flags); +size_t je_sallocx(const void *ptr, int flags); +void je_dallocx(void *ptr, int flags); +void je_sdallocx(void *ptr, size_t size, int flags); +size_t je_nallocx(size_t size, int flags); + +int je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +int je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp); +int je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen); +void je_malloc_stats_print(void (*write_cb)(void *, const char *), void *je_cbopaque, + const char *opts); +size_t je_malloc_usable_size(const void *ptr); + +void *je_memalign(size_t alignment, size_t size); +#if !defined(__ANDROID__) +void *je_valloc(size_t size); +#endif + +void *malloc(size_t size) { + return je_malloc(size); +} + +void *calloc(size_t num, size_t size) { + return je_calloc(num, size); +} + +int posix_memalign(void **memptr, size_t alignment, size_t size) { + return je_posix_memalign(memptr, alignment, size); +} + +void *aligned_alloc(size_t alignment, size_t size) { + return je_aligned_alloc(alignment, size); +} + +void *realloc(void *ptr, size_t size) { + return je_realloc(ptr, size); +} + +void free(void *ptr) { + je_free(ptr); +} + +void *mallocx(size_t size, int flags) { + return je_mallocx(size, flags); +} + +void *rallocx(void *ptr, size_t size, int flags) { + return je_rallocx(ptr, size, flags); +} + +size_t xallocx(void *ptr, size_t size, size_t extra, int flags) { + return je_xallocx(ptr, size, extra, flags); +} + +size_t sallocx(const void *ptr, int flags) { + return je_sallocx(ptr, flags); +} + +void dallocx(void *ptr, int flags) { + je_dallocx(ptr, flags); +} + +void sdallocx(void *ptr, size_t size, int flags) { + je_sdallocx(ptr, size, flags); +} + +size_t nallocx(size_t size, int flags) { + return je_nallocx(size, flags); +} + +int mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + return je_mallctl(name, oldp, oldlenp, newp, newlen); +} + +int mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { + return je_mallctlnametomib(name, mibp, miblenp); +} + +int mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { + return je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen); +} + +void malloc_stats_print(void (*write_cb)(void *, const char *), void *je_cbopaque, + const char *opts) { + return je_malloc_stats_print(write_cb, je_cbopaque, opts); +} + +size_t malloc_usable_size(const void *ptr) { + return je_malloc_usable_size(ptr); +} + +void *memalign(size_t alignment, size_t size) { + return je_memalign(alignment, size); +} + +void *valloc(size_t size) { + return je_valloc(size); +}