From fa23d4fe93020bfde8f45c7cad077117d999c3ed Mon Sep 17 00:00:00 2001 From: "Samuel E. Moelius III" Date: Sat, 18 Sep 2021 18:10:01 -0400 Subject: [PATCH 1/5] Implement #85440 --- library/test/src/cli.rs | 74 +++++++++++++++++++++++++++ library/test/src/console.rs | 4 +- library/test/src/event.rs | 2 +- library/test/src/formatters/json.rs | 11 ++-- library/test/src/formatters/junit.rs | 6 ++- library/test/src/formatters/mod.rs | 2 +- library/test/src/formatters/pretty.rs | 9 +++- library/test/src/formatters/terse.rs | 9 +++- library/test/src/helpers/mod.rs | 1 + library/test/src/helpers/shuffle.rs | 67 ++++++++++++++++++++++++ library/test/src/lib.rs | 11 +++- library/test/src/tests.rs | 2 + src/tools/compiletest/src/main.rs | 2 + 13 files changed, 186 insertions(+), 14 deletions(-) create mode 100644 library/test/src/helpers/shuffle.rs diff --git a/library/test/src/cli.rs b/library/test/src/cli.rs index 84874a2d2254a..e26b910101101 100644 --- a/library/test/src/cli.rs +++ b/library/test/src/cli.rs @@ -21,6 +21,8 @@ pub struct TestOpts { pub nocapture: bool, pub color: ColorConfig, pub format: OutputFormat, + pub shuffle: bool, + pub shuffle_seed: Option, pub test_threads: Option, pub skip: Vec, pub time_options: Option, @@ -138,6 +140,13 @@ fn optgroups() -> getopts::Options { `CRITICAL_TIME` here means the limit that should not be exceeded by test. ", + ) + .optflag("", "shuffle", "Run tests in random order") + .optopt( + "", + "shuffle-seed", + "Run tests in random order; seed the random number generator with SEED", + "SEED", ); opts } @@ -155,6 +164,12 @@ By default, all tests are run in parallel. This can be altered with the --test-threads flag or the RUST_TEST_THREADS environment variable when running tests (set it to 1). +By default, the tests are run in alphabetical order. Use --shuffle or set +RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated +"shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the +tests in the same order again. Note that --shuffle and --shuffle-seed do not +affect whether the tests are run in parallel. + All tests have their standard output and standard error captured by default. This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE environment variable to a value other than "0". Logging is not captured by default. @@ -218,6 +233,21 @@ macro_rules! unstable_optflag { }}; } +// Gets the option value and checks if unstable features are enabled. +macro_rules! unstable_optopt { + ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{ + let opt = $matches.opt_str($option_name); + if !$allow_unstable && opt.is_some() { + return Err(format!( + "The \"{}\" option is only accepted on the nightly compiler with -Z unstable-options", + $option_name + )); + } + + opt + }}; +} + // Implementation of `parse_opts` that doesn't care about help message // and returns a `Result`. fn parse_opts_impl(matches: getopts::Matches) -> OptRes { @@ -227,6 +257,8 @@ fn parse_opts_impl(matches: getopts::Matches) -> OptRes { let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process"); let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic"); let time_options = get_time_options(&matches, allow_unstable)?; + let shuffle = get_shuffle(&matches, allow_unstable)?; + let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?; let include_ignored = matches.opt_present("include-ignored"); let quiet = matches.opt_present("quiet"); @@ -260,6 +292,8 @@ fn parse_opts_impl(matches: getopts::Matches) -> OptRes { nocapture, color, format, + shuffle, + shuffle_seed, test_threads, skip, time_options, @@ -303,6 +337,46 @@ fn get_time_options( Ok(options) } +fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes { + let mut shuffle = unstable_optflag!(matches, allow_unstable, "shuffle"); + if !shuffle { + shuffle = match env::var("RUST_TEST_SHUFFLE") { + Ok(val) => &val != "0", + Err(_) => false, + }; + } + + Ok(shuffle) +} + +fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes> { + let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed") { + Some(n_str) => match n_str.parse::() { + Ok(n) => Some(n), + Err(e) => { + return Err(format!( + "argument for --shuffle-seed must be a number \ + (error: {})", + e + )); + } + }, + None => None, + }; + + if shuffle_seed.is_none() { + shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") { + Ok(val) => match val.parse::() { + Ok(n) => Some(n), + Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{}`, should be a number.", val), + }, + Err(_) => None, + }; + } + + Ok(shuffle_seed) +} + fn get_test_threads(matches: &getopts::Matches) -> OptPartRes> { let test_threads = match matches.opt_str("test-threads") { Some(n_str) => match n_str.parse::() { diff --git a/library/test/src/console.rs b/library/test/src/console.rs index 54e30a1fcd070..11c5ab48ed3e8 100644 --- a/library/test/src/console.rs +++ b/library/test/src/console.rs @@ -225,9 +225,9 @@ fn on_test_event( out: &mut dyn OutputFormatter, ) -> io::Result<()> { match (*event).clone() { - TestEvent::TeFiltered(ref filtered_tests) => { + TestEvent::TeFiltered(ref filtered_tests, shuffle_seed) => { st.total = filtered_tests.len(); - out.write_run_start(filtered_tests.len())?; + out.write_run_start(filtered_tests.len(), shuffle_seed)?; } TestEvent::TeFilteredOut(filtered_out) => { st.filtered_out = filtered_out; diff --git a/library/test/src/event.rs b/library/test/src/event.rs index 206f3e10e847d..6ff1a615eb4d0 100644 --- a/library/test/src/event.rs +++ b/library/test/src/event.rs @@ -28,7 +28,7 @@ impl CompletedTest { #[derive(Debug, Clone)] pub enum TestEvent { - TeFiltered(Vec), + TeFiltered(Vec, Option), TeWait(TestDesc), TeResult(CompletedTest), TeTimeout(TestDesc), diff --git a/library/test/src/formatters/json.rs b/library/test/src/formatters/json.rs index 57b6d1a02021f..424d3ef7b4106 100644 --- a/library/test/src/formatters/json.rs +++ b/library/test/src/formatters/json.rs @@ -60,10 +60,15 @@ impl JsonFormatter { } impl OutputFormatter for JsonFormatter { - fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()> { + let shuffle_seed_json = if let Some(shuffle_seed) = shuffle_seed { + format!(r#", "shuffle_seed": {}"#, shuffle_seed) + } else { + String::new() + }; self.writeln_message(&*format!( - r#"{{ "type": "suite", "event": "started", "test_count": {} }}"#, - test_count + r#"{{ "type": "suite", "event": "started", "test_count": {}{} }}"#, + test_count, shuffle_seed_json )) } diff --git a/library/test/src/formatters/junit.rs b/library/test/src/formatters/junit.rs index aa24480751419..f3ae5494906cc 100644 --- a/library/test/src/formatters/junit.rs +++ b/library/test/src/formatters/junit.rs @@ -27,7 +27,11 @@ impl JunitFormatter { } impl OutputFormatter for JunitFormatter { - fn write_run_start(&mut self, _test_count: usize) -> io::Result<()> { + fn write_run_start( + &mut self, + _test_count: usize, + _shuffle_seed: Option, + ) -> io::Result<()> { // We write xml header on run start self.write_message(&"") } diff --git a/library/test/src/formatters/mod.rs b/library/test/src/formatters/mod.rs index 2e03581b3af3a..cb80859759fad 100644 --- a/library/test/src/formatters/mod.rs +++ b/library/test/src/formatters/mod.rs @@ -18,7 +18,7 @@ pub(crate) use self::pretty::PrettyFormatter; pub(crate) use self::terse::TerseFormatter; pub(crate) trait OutputFormatter { - fn write_run_start(&mut self, test_count: usize) -> io::Result<()>; + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()>; fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; fn write_result( diff --git a/library/test/src/formatters/pretty.rs b/library/test/src/formatters/pretty.rs index 9cad71e30bddb..4a03b4b914760 100644 --- a/library/test/src/formatters/pretty.rs +++ b/library/test/src/formatters/pretty.rs @@ -181,9 +181,14 @@ impl PrettyFormatter { } impl OutputFormatter for PrettyFormatter { - fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()> { let noun = if test_count != 1 { "tests" } else { "test" }; - self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + let shuffle_seed_msg = if let Some(shuffle_seed) = shuffle_seed { + format!(" (shuffle seed: {})", shuffle_seed) + } else { + String::new() + }; + self.write_plain(&format!("\nrunning {} {}{}\n", test_count, noun, shuffle_seed_msg)) } fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { diff --git a/library/test/src/formatters/terse.rs b/library/test/src/formatters/terse.rs index 0c8215c5daca1..1f2c410cd96f3 100644 --- a/library/test/src/formatters/terse.rs +++ b/library/test/src/formatters/terse.rs @@ -170,10 +170,15 @@ impl TerseFormatter { } impl OutputFormatter for TerseFormatter { - fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option) -> io::Result<()> { self.total_test_count = test_count; let noun = if test_count != 1 { "tests" } else { "test" }; - self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + let shuffle_seed_msg = if let Some(shuffle_seed) = shuffle_seed { + format!(" (shuffle seed: {})", shuffle_seed) + } else { + String::new() + }; + self.write_plain(&format!("\nrunning {} {}{}\n", test_count, noun, shuffle_seed_msg)) } fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { diff --git a/library/test/src/helpers/mod.rs b/library/test/src/helpers/mod.rs index b7f00c4c86cdf..049cadf86a6d0 100644 --- a/library/test/src/helpers/mod.rs +++ b/library/test/src/helpers/mod.rs @@ -5,3 +5,4 @@ pub mod concurrency; pub mod exit_code; pub mod isatty; pub mod metrics; +pub mod shuffle; diff --git a/library/test/src/helpers/shuffle.rs b/library/test/src/helpers/shuffle.rs new file mode 100644 index 0000000000000..ca503106c556c --- /dev/null +++ b/library/test/src/helpers/shuffle.rs @@ -0,0 +1,67 @@ +use crate::cli::TestOpts; +use crate::types::{TestDescAndFn, TestId, TestName}; +use std::collections::hash_map::DefaultHasher; +use std::hash::Hasher; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub fn get_shuffle_seed(opts: &TestOpts) -> Option { + opts.shuffle_seed.or_else(|| { + if opts.shuffle { + Some( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Failed to get system time") + .as_nanos() as u64, + ) + } else { + None + } + }) +} + +pub fn shuffle_tests(shuffle_seed: u64, tests: &mut [(TestId, TestDescAndFn)]) { + let test_names: Vec<&TestName> = tests.iter().map(|test| &test.1.desc.name).collect(); + let test_names_hash = calculate_hash(&test_names); + let mut rng = Rng::new(shuffle_seed, test_names_hash); + shuffle(&mut rng, tests); +} + +// `shuffle` is from `rust-analyzer/src/cli/analysis_stats.rs`. +fn shuffle(rng: &mut Rng, slice: &mut [T]) { + for i in 0..slice.len() { + randomize_first(rng, &mut slice[i..]); + } + + fn randomize_first(rng: &mut Rng, slice: &mut [T]) { + assert!(!slice.is_empty()); + let idx = rng.rand_range(0..slice.len() as u64) as usize; + slice.swap(0, idx); + } +} + +struct Rng { + state: u64, + extra: u64, +} + +impl Rng { + fn new(seed: u64, extra: u64) -> Self { + Self { state: seed, extra } + } + + fn rand_range(&mut self, range: core::ops::Range) -> u64 { + self.rand_u64() % (range.end - range.start) + range.start + } + + fn rand_u64(&mut self) -> u64 { + self.state = calculate_hash(&(self.state, self.extra)); + self.state + } +} + +// `calculate_hash` is from `core/src/hash/mod.rs`. +fn calculate_hash(t: &T) -> u64 { + let mut s = DefaultHasher::new(); + t.hash(&mut s); + s.finish() +} diff --git a/library/test/src/lib.rs b/library/test/src/lib.rs index 251f099f28af4..67548819f6543 100644 --- a/library/test/src/lib.rs +++ b/library/test/src/lib.rs @@ -91,6 +91,7 @@ mod tests; use event::{CompletedTest, TestEvent}; use helpers::concurrency::get_concurrency; use helpers::exit_code::get_exit_code; +use helpers::shuffle::{get_shuffle_seed, shuffle_tests}; use options::{Concurrent, RunStrategy}; use test_result::*; use time::TestExecTime; @@ -247,7 +248,9 @@ where let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect(); - let event = TestEvent::TeFiltered(filtered_descs); + let shuffle_seed = get_shuffle_seed(opts); + + let event = TestEvent::TeFiltered(filtered_descs, shuffle_seed); notify_about_test_event(event)?; let (filtered_tests, filtered_benchs): (Vec<_>, _) = filtered_tests @@ -259,7 +262,11 @@ where let concurrency = opts.test_threads.unwrap_or_else(get_concurrency); let mut remaining = filtered_tests; - remaining.reverse(); + if let Some(shuffle_seed) = shuffle_seed { + shuffle_tests(shuffle_seed, &mut remaining); + } else { + remaining.reverse(); + } let mut pending = 0; let (tx, rx) = channel::(); diff --git a/library/test/src/tests.rs b/library/test/src/tests.rs index 794f727700476..7b68b8fbaf82f 100644 --- a/library/test/src/tests.rs +++ b/library/test/src/tests.rs @@ -45,6 +45,8 @@ impl TestOpts { nocapture: false, color: AutoColor, format: OutputFormat::Pretty, + shuffle: false, + shuffle_seed: None, test_threads: None, skip: vec![], time_options: None, diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 9e655557fd271..d72ed61d13401 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -505,6 +505,8 @@ pub fn test_opts(config: &Config) -> test::TestOpts { Err(_) => false, }, color: config.color, + shuffle: false, + shuffle_seed: None, test_threads: None, skip: vec![], list: false, From a6738c7231dadb0a84d338ae6f41cc34651518eb Mon Sep 17 00:00:00 2001 From: "Samuel E. Moelius III" Date: Sun, 19 Sep 2021 19:04:07 -0400 Subject: [PATCH 2/5] Add tests --- library/test/src/tests.rs | 117 ++++++++++++++++++++++++++++++-------- 1 file changed, 92 insertions(+), 25 deletions(-) diff --git a/library/test/src/tests.rs b/library/test/src/tests.rs index 7b68b8fbaf82f..718613895dee4 100644 --- a/library/test/src/tests.rs +++ b/library/test/src/tests.rs @@ -567,11 +567,7 @@ pub fn exact_filter_match() { assert_eq!(exact.len(), 2); } -#[test] -pub fn sort_tests() { - let mut opts = TestOpts::new(); - opts.run_tests = true; - +fn sample_tests() -> Vec { let names = vec![ "sha1::test".to_string(), "isize::test_to_str".to_string(), @@ -585,26 +581,32 @@ pub fn sort_tests() { "test::run_include_ignored_option".to_string(), "test::sort_tests".to_string(), ]; - let tests = { - fn testfn() {} - let mut tests = Vec::new(); - for name in &names { - let test = TestDescAndFn { - desc: TestDesc { - name: DynTestName((*name).clone()), - ignore: false, - should_panic: ShouldPanic::No, - allow_fail: false, - compile_fail: false, - no_run: false, - test_type: TestType::Unknown, - }, - testfn: DynTestFn(Box::new(testfn)), - }; - tests.push(test); - } - tests - }; + fn testfn() {} + let mut tests = Vec::new(); + for name in &names { + let test = TestDescAndFn { + desc: TestDesc { + name: DynTestName((*name).clone()), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + compile_fail: false, + no_run: false, + test_type: TestType::Unknown, + }, + testfn: DynTestFn(Box::new(testfn)), + }; + tests.push(test); + } + tests +} + +#[test] +pub fn sort_tests() { + let mut opts = TestOpts::new(); + opts.run_tests = true; + + let tests = sample_tests(); let filtered = filter_tests(&opts, tests); let expected = vec![ @@ -626,6 +628,71 @@ pub fn sort_tests() { } } +#[test] +pub fn shuffle_tests() { + let mut opts = TestOpts::new(); + opts.shuffle = true; + + let shuffle_seed = get_shuffle_seed(&opts).unwrap(); + + let left = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + let mut right = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + + assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name)); + + helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); + + assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name)); +} + +#[test] +pub fn shuffle_tests_with_seed() { + let mut opts = TestOpts::new(); + opts.shuffle = true; + + let shuffle_seed = get_shuffle_seed(&opts).unwrap(); + + let mut left = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + let mut right = + sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + + helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice()); + helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); + + assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name)); +} + +#[test] +pub fn order_depends_on_more_than_seed() { + let mut opts = TestOpts::new(); + opts.shuffle = true; + + let shuffle_seed = get_shuffle_seed(&opts).unwrap(); + + let mut left_tests = sample_tests(); + let mut right_tests = sample_tests(); + + left_tests.pop(); + right_tests.remove(0); + + let mut left = + left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + let mut right = + right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::>(); + + assert_eq!(left.len(), right.len()); + + assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0)); + + helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice()); + helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); + + assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0)); +} + #[test] pub fn test_metricmap_compare() { let mut m1 = MetricMap::new(); From 32b6ac5b44e4f0ffa5e56b9c81407633dc4edb64 Mon Sep 17 00:00:00 2001 From: "Samuel E. Moelius III" Date: Thu, 30 Sep 2021 12:57:34 -0400 Subject: [PATCH 3/5] Check `allow_unstable` before checking environment variables --- library/test/src/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/test/src/cli.rs b/library/test/src/cli.rs index e26b910101101..cb40b4e965b2a 100644 --- a/library/test/src/cli.rs +++ b/library/test/src/cli.rs @@ -339,7 +339,7 @@ fn get_time_options( fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes { let mut shuffle = unstable_optflag!(matches, allow_unstable, "shuffle"); - if !shuffle { + if !shuffle && allow_unstable { shuffle = match env::var("RUST_TEST_SHUFFLE") { Ok(val) => &val != "0", Err(_) => false, @@ -364,7 +364,7 @@ fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPart None => None, }; - if shuffle_seed.is_none() { + if shuffle_seed.is_none() && allow_unstable { shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") { Ok(val) => match val.parse::() { Ok(n) => Some(n), From e16e15f3aec7437181080b8a5025b42a26436565 Mon Sep 17 00:00:00 2001 From: "Samuel E. Moelius III" Date: Mon, 4 Oct 2021 18:53:22 -0400 Subject: [PATCH 4/5] Add documentation --- src/doc/rustc/src/tests/index.md | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/doc/rustc/src/tests/index.md b/src/doc/rustc/src/tests/index.md index ec23d4fe0dbd3..2681105b468d5 100644 --- a/src/doc/rustc/src/tests/index.md +++ b/src/doc/rustc/src/tests/index.md @@ -181,6 +181,38 @@ unstable-options` flag. See [tracking issue #64888](https://github.com/rust-lang/rust/issues/64888) and the [unstable docs](../../unstable-book/compiler-flags/report-time.html) for more information. +#### `--shuffle` + +Runs the tests in random order, as opposed to the default alphabetical order. + +This may also be specified by setting the `RUST_TEST_SHUFFLE` environment +variable to anything but `0`. + +The random number generator seed that is output can be passed to +[`--shuffle-seed`](#--shuffle-seed-seed) to run the tests in the same order +again. + +Note that `--shuffle` does not affect whether the tests are run in parallel. To +run the tests in random order sequentially, use `--shuffle --test-threads 1`. + +⚠️ 🚧 This option is [unstable](#unstable-options), and requires the `-Z +unstable-options` flag. + +#### `--shuffle-seed` _SEED_ + +Like [`--shuffle`](#--shuffle), but seeds the random number generator with +_SEED_. Thus, calling the test harness with `--shuffle-seed` _SEED_ twice runs +the tests in the same order both times. + +_SEED_ is any 64-bit unsigned integer, for example, one produced by +[`--shuffle`](#--shuffle). + +This can also be specified with the `RUST_TEST_SHUFFLE_SEED` environment +variable. + +⚠️ 🚧 This option is [unstable](#unstable-options), and requires the `-Z +unstable-options` flag. + ### Output options The following options affect the output behavior. @@ -197,7 +229,7 @@ to the console. Usually the output is captured, and only displayed if the test fails. This may also be specified by setting the `RUST_TEST_NOCAPTURE` environment -variable set to anything but `0`. +variable to anything but `0`. #### `--show-output` From ecf474152350664f1227421eeb278b2e8185cd07 Mon Sep 17 00:00:00 2001 From: "Samuel E. Moelius III" Date: Tue, 5 Oct 2021 20:46:28 -0400 Subject: [PATCH 5/5] Add tracking issue --- src/doc/rustc/src/tests/index.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/doc/rustc/src/tests/index.md b/src/doc/rustc/src/tests/index.md index 2681105b468d5..66e3a5261e4bd 100644 --- a/src/doc/rustc/src/tests/index.md +++ b/src/doc/rustc/src/tests/index.md @@ -196,7 +196,8 @@ Note that `--shuffle` does not affect whether the tests are run in parallel. To run the tests in random order sequentially, use `--shuffle --test-threads 1`. ⚠️ 🚧 This option is [unstable](#unstable-options), and requires the `-Z -unstable-options` flag. +unstable-options` flag. See [tracking issue +#89583](https://github.com/rust-lang/rust/issues/89583) for more information. #### `--shuffle-seed` _SEED_ @@ -211,7 +212,8 @@ This can also be specified with the `RUST_TEST_SHUFFLE_SEED` environment variable. ⚠️ 🚧 This option is [unstable](#unstable-options), and requires the `-Z -unstable-options` flag. +unstable-options` flag. See [tracking issue +#89583](https://github.com/rust-lang/rust/issues/89583) for more information. ### Output options