wasm: Store rlib metadata in wasm object files

The goal of this commit is to remove warnings using LLVM tip-of-tree
`wasm-ld`. In llvm/llvm-project#78658 the `wasm-ld` LLD driver no longer
looks at archive indices and instead looks at all the objects in
archives. Previously `lib.rmeta` files were simply raw rustc metadata
bytes, not wasm objects, meaning that `wasm-ld` would emit a warning
indicating so.

WebAssembly targets previously passed `--fatal-warnings` to `wasm-ld` by
default which meant that if Rust were to update to LLVM 18 then all wasm
targets would not work. This immediate blocker was resolved in
#120278 which removed `--fatal-warnings` which enabled a
theoretical update to LLVM 18 for wasm targets. This current state is
ok-enough for now because rustc squashes all linker output by default if
it doesn't fail. This means, for example, that rustc squashes all the
linker warnings coming out of `wasm-ld` about `lib.rmeta` files with
LLVM 18. This again isn't a pressing issue because the information is
all hidden, but it runs the risk of being annoying if another linker
error were to happen and then the output would have all these unrelated
warnings that couldn't be fixed.

Thus, this PR comes into the picture. The goal of this PR is to resolve
these warnings by using the WebAssembly object file format on wasm
targets instead of using raw rustc metadata. When I first implemented
the rlib-in-objects scheme in #84449 I remember either concluding that
`wasm-ld` would either include the metadata in the output or I thought
we didn't have to do anything there at all. I think I was wrong on both
counts as `wasm-ld` does not include the metadata in the final output
unless the object is referenced and we do actually need to do something
to resolve these warnings.

This PR updates the object file format containing rustc metadata on
WebAssembly targets to be an actual WebAssembly file. To avoid bringing
in any new dependencies I've opted to hand-code this encoding at this
time. If the object gets more complicated though it'd probably be best
to pull in `wasmparser` and `wasm-encoder`. For now though there's two
adjacent functions reading/writing wasm.

The only caveat I know of with this is that if `wasm-ld` does indeed
look at the object file then the metadata will be included in the final
output. I believe the only thing that could cause that at this time is
`--whole-archive` which I don't think is passed for rlibs. I would
clarify that I'm not 100% certain about this, however.

Author: alexcrichton

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -15,6 +15,8 @@ use rustc_data_structures::owned_slice::{try_slice_owned, OwnedSlice};
 use rustc_metadata::creader::MetadataLoader;
 use rustc_metadata::fs::METADATA_FILENAME;
 use rustc_metadata::EncodedMetadata;
+use rustc_serialize::leb128;
+use rustc_serialize::{opaque::MemDecoder, Decoder};
 use rustc_session::Session;
 use rustc_span::sym;
 use rustc_target::abi::Endian;
@@ -62,6 +64,8 @@ impl MetadataLoader for DefaultMetadataLoader {
                         .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?;
                     if target.is_like_aix {
                         return get_metadata_xcoff(path, data);
+                    } else if target.is_like_wasm {
+                        return get_metadata_wasm(data, ".rmeta");
                     } else {
                         return search_for_section(path, data, ".rmeta");
                     }
@@ -420,10 +424,9 @@ pub enum MetadataPosition {
 ///   it's not in an allowlist of otherwise well known dwarf section names to
 ///   go into the final artifact.
 ///
-/// * WebAssembly - we actually don't have any container format for this
-///   target. WebAssembly doesn't support the `dylib` crate type anyway so
-///   there's no need for us to support this at this time. Consequently the
-///   metadata bytes are simply stored as-is into an rlib.
+/// * WebAssembly - this uses wasm files themselves as the object file format
+///   so an empty file with no linking metadata but a single custom section is
+///   created holding our metadata.
 ///
 /// * COFF - Windows-like targets create an object with a section that has
 ///   the `IMAGE_SCN_LNK_REMOVE` flag set which ensures that if the linker
@@ -438,22 +441,13 @@ pub fn create_wrapper_file(
     data: &[u8],
 ) -> (Vec<u8>, MetadataPosition) {
     let Some(mut file) = create_object_file(sess) else {
-        // This is used to handle all "other" targets. This includes targets
-        // in two categories:
-        //
-        // * Some targets don't have support in the `object` crate just yet
-        //   to write an object file. These targets are likely to get filled
-        //   out over time.
-        //
-        // * Targets like WebAssembly don't support dylibs, so the purpose
-        //   of putting metadata in object files, to support linking rlibs
-        //   into dylibs, is moot.
-        //
-        // In both of these cases it means that linking into dylibs will
-        // not be supported by rustc. This doesn't matter for targets like
-        // WebAssembly and for targets not supported by the `object` crate
-        // yet it means that work will need to be done in the `object` crate
-        // to add a case above.
+        if sess.target.is_like_wasm {
+            return (create_metadata_file_for_wasm(data, &section_name), MetadataPosition::First);
+        }
+
+        // Targets using this branch don't have support implemented here yet or
+        // they're not yet implemented in the `object` crate and will likely
+        // fill out this module over time.
         return (data.to_vec(), MetadataPosition::Last);
     };
     let section = if file.format() == BinaryFormat::Xcoff {
@@ -532,6 +526,9 @@ pub fn create_compressed_metadata_file(
     packed_metadata.extend(metadata.raw_data());
 
     let Some(mut file) = create_object_file(sess) else {
+        if sess.target.is_like_wasm {
+            return create_metadata_file_for_wasm(&packed_metadata, b".rustc");
+        }
         return packed_metadata.to_vec();
     };
     if file.format() == BinaryFormat::Xcoff {
@@ -624,3 +621,85 @@ pub fn create_compressed_metadata_file_for_xcoff(
     file.append_section_data(section, data, 1);
     file.write().unwrap()
 }
+
+/// Creates a simple WebAssembly object file, which is itself a wasm module,
+/// that contains a custom section of the name `section_name` with contents
+/// `data`.
+///
+/// NB: the wasm file format is simple enough that for now an extra crate from
+/// crates.io (such as `wasm-encoder` isn't used at this time (nor `wasmparser`
+/// for example to parse). The file format is:
+///
+/// * 4-byte header "\0asm"
+/// * 4-byte version number - 1u32 in little-endian format
+/// * concatenated sections, which for this object is always "custom sections"
+///
+/// Custom sections are then defiend by:
+/// * 1-byte section identifier - 0 for a custom section
+/// * leb-encoded section length (size of the contents beneath this bullet)
+///   * leb-encoded custom section name length
+///   * custom section name
+///   * section contents
+///
+/// One custom section, `linking`, is added here in accordance with
+/// <https://github.com/WebAssembly/tool-conventions/blob/main/Linking.md>
+/// which is required to inform LLD that this is an object file but it should
+/// otherwise basically ignore it if it otherwise looks at it. The linking
+/// section currently is defined by a single verion byte (2) and then further
+/// sections, but we have no more sections, so it's just the byte "2".
+///
+/// The next custom section is the one we're interested in.
+pub fn create_metadata_file_for_wasm(data: &[u8], section_name: &[u8]) -> Vec<u8> {
+    let mut bytes = b"\0asm\x01\0\0\0".to_vec();
+
+    let mut append_custom_section = |section_name: &[u8], data: &[u8]| {
+        let mut section_name_len = [0; leb128::max_leb128_len::<usize>()];
+        let off = leb128::write_usize_leb128(&mut section_name_len, section_name.len());
+        let section_name_len = &section_name_len[..off];
+
+        let mut section_len = [0; leb128::max_leb128_len::<usize>()];
+        let off = leb128::write_usize_leb128(
+            &mut section_len,
+            data.len() + section_name_len.len() + section_name.len(),
+        );
+        let section_len = &section_len[..off];
+
+        bytes.push(0u8);
+        bytes.extend_from_slice(section_len);
+        bytes.extend_from_slice(section_name_len);
+        bytes.extend_from_slice(section_name);
+        bytes.extend_from_slice(data);
+    };
+
+    append_custom_section(b"linking", &[2]);
+    append_custom_section(section_name, data);
+    bytes
+}
+
+// NB: see documentation on `create_metadata_file_for_wasm` above for
+// particulars on the wasm format.
+fn get_metadata_wasm<'a>(data: &'a [u8], expected_section_name: &str) -> Result<&'a [u8], String> {
+    let data = data
+        .strip_prefix(b"\0asm\x01\0\0\0")
+        .ok_or_else(|| format!("metadata has an invalid wasm header"))?;
+
+    let mut decoder = MemDecoder::new(data, 0);
+    let mut next_custom_section = |expected_section_name: &str| {
+        if decoder.read_u8() != 0 {
+            return Err(format!("metadata did not start with a custom section"));
+        }
+        let section_len = leb128::read_usize_leb128(&mut decoder);
+
+        let section_start = decoder.position();
+        let mut section = MemDecoder::new(decoder.read_raw_bytes(section_len), 0);
+        let name_len = leb128::read_usize_leb128(&mut section);
+        let section_name = section.read_raw_bytes(name_len);
+        if section_name != expected_section_name.as_bytes() {
+            return Err(format!("unexpected section name in metadata object"));
+        }
+        Ok(&data[section_start + section.position()..][..section.remaining()])
+    };
+
+    next_custom_section("linking")?;
+    next_custom_section(expected_section_name)
+}