From b15b55b8fc4879ad505d1b1853814a7604c876e0 Mon Sep 17 00:00:00 2001 From: Yamakaky Date: Wed, 8 Feb 2017 19:32:32 -0500 Subject: [PATCH 1/3] Divide test run between multiple travis builds. --- .travis.yml | 5 ++++- src/bootstrap/flags.rs | 18 ++++++++++++++++++ src/bootstrap/step.rs | 28 ++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a776f1b1e1041..f88c598616855 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,10 @@ matrix: fast_finish: true include: # Linux builders, all docker images + - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1 BOOTSTRAP_ARGS="--slice 0 --number-slices 4" + - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1 BOOTSTRAP_ARGS="--slice 1 --number-slices 4" + - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1 BOOTSTRAP_ARGS="--slice 2 --number-slices 4" + - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1 BOOTSTRAP_ARGS="--slice 3 --number-slices 4" - env: IMAGE=android DEPLOY=1 - env: IMAGE=armhf-gnu - env: IMAGE=cross DEPLOY=1 @@ -33,7 +37,6 @@ matrix: - env: IMAGE=x86_64-gnu-aux - env: IMAGE=x86_64-gnu-debug - env: IMAGE=x86_64-gnu-nopt - - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1 - env: IMAGE=x86_64-gnu-distcheck - env: IMAGE=x86_64-gnu-incremental diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index c5bbfd89b2787..9b0de9233ea50 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -60,6 +60,7 @@ pub enum Subcommand { Test { paths: Vec, test_args: Vec, + split: Option<(usize, usize)>, }, Bench { paths: Vec, @@ -224,10 +225,27 @@ To learn more about a subcommand, run `./x.py -h` } "test" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); + opts.optopt("", + "number-slices", + "divide the test run in N segments", + "N"); + opts.optopt("", + "slice", + "run ith segment", + "i"); m = parse(&opts); + let split = match (m.opt_str("slice"), m.opt_str("number-slices")) { + (Some(raw_slice), Some(raw_nb_slices)) => { + Some((raw_slice.parse().unwrap(), + raw_nb_slices.parse().unwrap())) + } + (None, None) => None, + _ => panic!("You must provide --slice and --number-slices"), + }; Subcommand::Test { paths: remaining_as_path(&m), test_args: m.opt_strs("test-args"), + split: split, } } "bench" => { diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index ee5b61062fed8..7b086f0bb948f 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -28,6 +28,7 @@ use std::collections::{HashMap, HashSet}; use std::mem; +use std::cmp::min; use check::{self, TestKind}; use compile; @@ -982,10 +983,14 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? // product of the two and then create a step based off them. Note that // the stage each step is associated was specified with the `--step` // flag on the command line. + let mut split = None; let (kind, paths) = match self.build.flags.cmd { Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]), - Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]), + Subcommand::Test { ref paths, test_args: _, split: new_split } => { + split = new_split; + (Kind::Test, &paths[..]) + } Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]), Subcommand::Dist { ref paths, install } => { if install { @@ -996,8 +1001,7 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? } Subcommand::Clean => panic!(), }; - - self.rules.values().filter(|rule| rule.kind == kind).filter(|rule| { + let steps = self.rules.values().filter(|rule| rule.kind == kind).filter(|rule| { (paths.len() == 0 && rule.default) || paths.iter().any(|path| { path.ends_with(rule.path) }) @@ -1041,7 +1045,23 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? self.sbuild.name(rule.name).target(target).host(host) }) }) - }).collect() + }).collect::>(); + + if let Some((offset, nb_split)) = split { + assert!(nb_split <= steps.len(), + "you asked for {} slices but there are only {} tasks", + nb_split, + steps.len()); + let index = |i| { + i * (steps.len() / nb_split) + min(steps.len() % nb_split, i) + }; + let from = index(offset); + let to = index(offset + 1); + steps[from..to].into() + } else { + steps + } + } /// Execute all top-level targets indicated by `steps`. From 234d2bed148ca21acf34d4f23e4a9f57a0cfe2e7 Mon Sep 17 00:00:00 2001 From: Yamakaky Date: Thu, 9 Feb 2017 14:41:34 -0500 Subject: [PATCH 2/3] Time each command --- src/ci/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ci/run.sh b/src/ci/run.sh index 960acc4de7d87..1e458e8162605 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -67,7 +67,7 @@ fi if [ ! -z "$SCRIPT" ]; then sh -x -c "$SCRIPT" else - make -j $ncpus tidy - make -j $ncpus - make $RUST_CHECK_TARGET -j $ncpus + time make -j $ncpus tidy + time make -j $ncpus + time make $RUST_CHECK_TARGET -j $ncpus fi From 079ffd0ee2bffdab9c72a18c4478c7820ef43ae8 Mon Sep 17 00:00:00 2001 From: Yamakaky Date: Thu, 9 Feb 2017 21:30:51 -0500 Subject: [PATCH 3/3] Use bash for builtin `time` --- src/ci/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/run.sh b/src/ci/run.sh index 1e458e8162605..0eeeaa998b62a 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2016 The Rust Project Developers. See the COPYRIGHT # file at the top-level directory of this distribution and at # http://rust-lang.org/COPYRIGHT.