Symlink all transitive crates into a single directory on Windows

Transitive crate dependencies are passed to rustc by specifying the
folder they live in as a library search path (an `-Ldependency=path`
flag). Given that bazel builds all libraries into their own folders,
this means passing potentially hundreds of paths to rustc which can lead
to overflowing the rather short limit for command arguments on Windows.

For this reason, this patch creates symlinks to all the transitive
crates into a single directory that it then sets as the `-Ldependency=`
flag to rustc. This dramatically shortens the command arguments length
when calling rustc. Note that we only do that on Windows since it
introduces some prep actions that need to do IO and there is no need to
regress other platforms that support much longer commane args.

I opted for declaring these files as inputs and creating the symlinks
via `ctx.actions.symlink` but another option would have been to do that
in the process wrapper instead.

For context, here is the size returned by `getconf ARG_MAX` on my
following machines:
```
macos (12 x86_64):      1,048,576
linux (arch x86_64):    2,097,152  // obviously somewhat lower on 32-bit but
I don't have a machine handy
windows (10 x86_64):    32,000
```

Author: ddeville

rust/private/clippy.bzl

@@ -73,7 +73,7 @@ def _clippy_aspect_impl(target, ctx):
         remove_transitive_libs_from_dep_info = toolchain._incompatible_remove_transitive_libs_from_dep_info,
     )
 
-    compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs = collect_inputs(
+    compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs, transitive_crates_dir = collect_inputs(
         ctx,
         ctx.rule.file,
         ctx.rule.files,
@@ -103,6 +103,7 @@ def _clippy_aspect_impl(target, ctx):
         build_env_files = build_env_files,
         build_flags_files = build_flags_files,
         emit = ["dep-info", "metadata"],
+        transitive_crates_dir = transitive_crates_dir,
     )
 
     if crate_info.is_test:

rust/private/rustc.bzl

@@ -374,6 +374,8 @@ def collect_inputs(
             - (File): An optional path to a generated environment file from a `cargo_build_script` target
             - (list): All direct and transitive build flags from the current build info
             - (list[File]): Linkstamp outputs.
+            - (File): Optional - the output directory containing all transitive crates that this crate depends on.
+                Only used on Windows. If it's None, all transitive crate search path will be specified individually.
     """
     linker_script = getattr(file, "linker_script") if hasattr(file, "linker_script") else None
 
@@ -449,10 +451,19 @@ def collect_inputs(
     # If stamping is enabled include the volatile status info file
     stamp_info = [ctx.version_file] if stamp else []
 
+    # Windows is known to have a small limit to the length the command args can have (32k characters).
+    # To prevent the arguments to rustc from overflowing this length, rather than adding each transitive dep as a single library
+    # search path flag, symlink all transitive deps into a single directory that we can pass as a single path flag.
+    if toolchain.os == "windows":
+        transitive_crates_dir, transitive_crates_links = symlink_transitive_crates(ctx, crate_info, dep_info)
+    else:
+        transitive_crates_dir, transitive_crates_links = None, []
+
     compile_inputs = depset(
         linkstamp_outs + stamp_info,
         transitive = [
             nolinkstamp_compile_inputs,
+            depset(transitive_crates_links),
         ],
     )
 
@@ -462,7 +473,7 @@ def collect_inputs(
         build_env_files = [f for f in build_env_files] + [build_env_file]
     compile_inputs = depset(build_env_files, transitive = [compile_inputs])
 
-    return compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs
+    return compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs, transitive_crates_dir
 
 def construct_arguments(
         ctx,
@@ -481,6 +492,7 @@ def construct_arguments(
         build_env_files,
         build_flags_files,
         emit = ["dep-info", "link"],
+        transitive_crates_dir = None,
         force_all_deps_direct = False,
         force_link = False,
         stamp = False,
@@ -504,6 +516,7 @@ def construct_arguments(
         build_env_files (list): Files containing rustc environment variables, for instance from `cargo_build_script` actions.
         build_flags_files (list): The output files of a `cargo_build_script` actions containing rustc build flags
         emit (list): Values for the --emit flag to rustc.
+        transitive_crates_dir (File, optional): The output directory containing all transitive crates that this crate depends on.
         force_all_deps_direct (bool, optional): Whether to pass the transitive rlibs with --extern
             to the commandline as opposed to -L.
         force_link (bool, optional): Whether to add link flags to the command regardless of `emit`.
@@ -650,7 +663,7 @@ def construct_arguments(
         _add_native_link_flags(rustc_flags, dep_info, linkstamp_outs, crate_info.type, toolchain, cc_toolchain, feature_configuration)
 
     # These always need to be added, even if not linking this crate.
-    add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct)
+    add_crate_link_flags(rustc_flags, dep_info, transitive_crates_dir, force_all_deps_direct)
 
     needs_extern_proc_macro_flag = "proc-macro" in [crate_info.type, crate_info.wrapped_crate_type] and \
                                    crate_info.edition != "2015"
@@ -739,7 +752,7 @@ def rustc_compile_action(
     # Determine if the build is currently running with --stamp
     stamp = is_stamping_enabled(attr)
 
-    compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs = collect_inputs(
+    compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs, transitive_crates_dir = collect_inputs(
         ctx = ctx,
         file = ctx.file,
         files = ctx.files,
@@ -769,6 +782,7 @@ def rustc_compile_action(
         out_dir = out_dir,
         build_env_files = build_env_files,
         build_flags_files = build_flags_files,
+        transitive_crates_dir = transitive_crates_dir,
         force_all_deps_direct = force_all_deps_direct,
         stamp = stamp,
     )
@@ -1040,12 +1054,54 @@ def _get_dir_names(files):
         dirs[f.dirname] = None
     return dirs.keys()
 
-def add_crate_link_flags(args, dep_info, force_all_deps_direct = False):
+def symlink_transitive_crates(ctx, crate_info, dep_info):
+    """Collect and symlink the transitive crates into a single directory.
+
+    The reason for this is so that we can pass a single entry for the library search path, which greatly shortens
+    the length of the command line args that we pass to `rustc` and and prevents it from overflowing the limit set
+    by the host platform (which is really only a concern on Windows).
+
+    Args:
+        ctx (ctx): The rule's context object
+        crate_info (CrateInfo): The CrateInfo provider of the target crate
+        dep_info (DepInfo): The current target's dependency info
+
+    Returns:
+        tuple: A tuple of the following items
+            - (File): Optional - the output directory containing all transitive crates that this crate depends on.
+            - (list): The list of transitive crates files that should be used as input to the build action.
+    """
+    deps = dep_info.transitive_crates.to_list()
+    if not deps:
+        return None, []
+
+    output_dirname = crate_info.output.basename + ".transitive_crates"
+    links = []
+
+    # Keep a list of the crates that were currently added so that we can uniquify them.
+    names = {}
+
+    for dep in deps:
+        name = dep.output.basename
+        if name in names:
+            continue
+
+        link = ctx.actions.declare_file(output_dirname + "/" + name)
+        ctx.actions.symlink(output = link, target_file = dep.output, is_executable = True)
+
+        names[name] = True
+        links.append(link)
+
+    return links[0].dirname, links
+
+def add_crate_link_flags(args, dep_info, transitive_crates_dir = None, force_all_deps_direct = False):
     """Adds link flags to an Args object reference
 
     Args:
         args (Args): An arguments object reference
         dep_info (DepInfo): The current target's dependency info
+        transitive_crates_dir (File, optional): An output directory containing all transitive crates that this crate depends on.
+            If this argument is set, only it will be added as a `-Ldependency=` flag, otherwise all rlibs will be set individually.
         force_all_deps_direct (bool, optional): Whether to pass the transitive rlibs with --extern
             to the commandline as opposed to -L.
     """
@@ -1064,12 +1120,17 @@ def add_crate_link_flags(args, dep_info, force_all_deps_direct = False):
     else:
         # nb. Direct crates are linked via --extern regardless of their crate_type
         args.add_all(dep_info.direct_crates, map_each = _crate_to_link_flag)
-    args.add_all(
-        dep_info.transitive_crates,
-        map_each = _get_crate_dirname,
-        uniquify = True,
-        format_each = "-Ldependency=%s",
-    )
+
+    # If transitive rlibs have been collected into this single directory, only set this directory
+    if transitive_crates_dir:
+        args.add("-Ldependency={}".format(transitive_crates_dir))
+    else:
+        args.add_all(
+            dep_info.transitive_crates,
+            map_each = _get_crate_dirname,
+            uniquify = True,
+            format_each = "-Ldependency=%s",
+        )
 
 def _crate_to_link_flag(crate):
     """A helper macro used by `add_crate_link_flags` for adding crate link flags to a Arg object
@@ -1091,11 +1152,10 @@ def _crate_to_link_flag(crate):
     return ["--extern={}={}".format(name, crate_info.output.path)]
 
 def _get_crate_dirname(crate):
-    """A helper macro used by `add_crate_link_flags` for getting the directory name of the current crate's output path
+    """A helper macro used by `add_crate_link_flags` for getting the directory name of the current crate's output path.
 
     Args:
         crate (CrateInfo): A CrateInfo provider from the current rule
-
     Returns:
         str: The directory name of the the output File that will be produced.
     """

rust/private/rustdoc.bzl

@@ -79,7 +79,7 @@ def rustdoc_compile_action(
         remove_transitive_libs_from_dep_info = toolchain._incompatible_remove_transitive_libs_from_dep_info,
     )
 
-    compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs = collect_inputs(
+    compile_inputs, out_dir, build_env_files, build_flags_files, linkstamp_outs, transitive_crates_dir = collect_inputs(
         ctx = ctx,
         file = ctx.file,
         files = ctx.files,
@@ -115,6 +115,7 @@ def rustdoc_compile_action(
         build_env_files = build_env_files,
         build_flags_files = build_flags_files,
         emit = [],
+        transitive_crates_dir = transitive_crates_dir,
         remap_path_prefix = None,
         force_link = True,
     )

test/unit/common.bzl

@@ -62,6 +62,23 @@ def assert_action_mnemonic(env, action, mnemonic):
             ),
         )
 
+def find_actions_with_mnemonic(env, target, mnemonic):
+    """Find all the actions in the given target that have the given mnemonic.
+
+    Args:
+      env: env from analysistest.begin(ctx).
+      target: the target current being analyzed.
+      mnemonic (str): The action mnemonic we're filtering on.
+
+    Returns:
+        list: The actions that have the given mnemonic.
+    """
+    actions = []
+    for action in target.actions:
+        if action.mnemonic == mnemonic:
+            actions.append(action)
+    return actions
+
 def _startswith(list, prefix):
     if len(list) < len(prefix):
         return False