Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit d3508a8

Browse files
committed
Auto merge of #140177 - tmandry:compiletest-par, r=jieyouxu
[compiletest] Parallelize test discovery Certain filesystems are slow to service individual read requests, but can service many in parallel. This change brings down the time to run a single cached test on one of those filesystems from 40s to about 8s.
2 parents 555e1d0 + f673c9b commit d3508a8

File tree

4 files changed

+62
-40
lines changed

4 files changed

+62
-40
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,7 @@ dependencies = [
737737
"libc",
738738
"miow",
739739
"miropt-test-tools",
740+
"rayon",
740741
"regex",
741742
"rustfix",
742743
"semver",

‎compiler/rustc_mir_transform/src/coverage/mod.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
pub(super) mod query;
2-
31
mod counters;
42
mod graph;
53
mod mappings;
4+
pub(super) mod query;
65
mod spans;
76
#[cfg(test)]
87
mod tests;

‎src/tools/compiletest/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ glob = "0.3.0"
1818
home = "0.5.5"
1919
indexmap = "2.0.0"
2020
miropt-test-tools = { path = "../miropt-test-tools" }
21+
rayon = "1.10.0"
2122
regex = "1.0"
2223
rustfix = "0.8.1"
2324
semver = { version = "1.0.23", features = ["serde"] }

‎src/tools/compiletest/src/lib.rs‎

Lines changed: 59 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ use std::{env, fs, vec};
3333
use build_helper::git::{get_git_modified_files, get_git_untracked_files};
3434
use camino::{Utf8Path, Utf8PathBuf};
3535
use getopts::Options;
36+
use rayon::iter::{ParallelBridge, ParallelIterator};
3637
use tracing::*;
3738
use walkdir::WalkDir;
3839

@@ -638,6 +639,18 @@ struct TestCollector {
638639
poisoned: bool,
639640
}
640641

642+
impl TestCollector {
643+
fn new() -> Self {
644+
TestCollector { tests: vec![], found_path_stems: HashSet::new(), poisoned: false }
645+
}
646+
647+
fn merge(&mut self, mut other: Self) {
648+
self.tests.append(&mut other.tests);
649+
self.found_path_stems.extend(other.found_path_stems);
650+
self.poisoned |= other.poisoned;
651+
}
652+
}
653+
641654
/// Creates test structures for every test/revision in the test suite directory.
642655
///
643656
/// This always inspects _all_ test files in the suite (e.g. all 17k+ ui tests),
@@ -656,10 +669,7 @@ pub(crate) fn collect_and_make_tests(config: Arc<Config>) -> Vec<CollectedTest>
656669
let cache = HeadersCache::load(&config);
657670

658671
let cx = TestCollectorCx { config, cache, common_inputs_stamp, modified_tests };
659-
let mut collector =
660-
TestCollector { tests: vec![], found_path_stems: HashSet::new(), poisoned: false };
661-
662-
collect_tests_from_dir(&cx, &mut collector, &cx.config.src_test_suite_root, Utf8Path::new(""))
672+
let collector = collect_tests_from_dir(&cx, &cx.config.src_test_suite_root, Utf8Path::new(""))
663673
.unwrap_or_else(|reason| {
664674
panic!("Could not read tests from {}: {reason}", cx.config.src_test_suite_root)
665675
});
@@ -765,25 +775,25 @@ fn modified_tests(config: &Config, dir: &Utf8Path) -> Result<Vec<Utf8PathBuf>, S
765775
/// that will be handed over to libtest.
766776
fn collect_tests_from_dir(
767777
cx: &TestCollectorCx,
768-
collector: &mut TestCollector,
769778
dir: &Utf8Path,
770779
relative_dir_path: &Utf8Path,
771-
) -> io::Result<()> {
780+
) -> io::Result<TestCollector> {
772781
// Ignore directories that contain a file named `compiletest-ignore-dir`.
773782
if dir.join("compiletest-ignore-dir").exists() {
774-
return Ok(());
783+
return Ok(TestCollector::new());
775784
}
776785

777786
// For run-make tests, a "test file" is actually a directory that contains an `rmake.rs`.
778787
if cx.config.mode == Mode::RunMake {
788+
let mut collector = TestCollector::new();
779789
if dir.join("rmake.rs").exists() {
780790
let paths = TestPaths {
781791
file: dir.to_path_buf(),
782792
relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
783793
};
784-
make_test(cx, collector, &paths);
794+
make_test(cx, &mutcollector, &paths);
785795
// This directory is a test, so don't try to find other tests inside it.
786-
return Ok(());
796+
return Ok(collector);
787797
}
788798
}
789799

@@ -800,36 +810,47 @@ fn collect_tests_from_dir(
800810
// subdirectories we find, except for `auxiliary` directories.
801811
// FIXME: this walks full tests tree, even if we have something to ignore
802812
// use walkdir/ignore like in tidy?
803-
for file in fs::read_dir(dir.as_std_path())? {
804-
let file = file?;
805-
let file_path = Utf8PathBuf::try_from(file.path()).unwrap();
806-
let file_name = file_path.file_name().unwrap();
807-
808-
if is_test(file_name)
809-
&& (!cx.config.only_modified || cx.modified_tests.contains(&file_path))
810-
{
811-
// We found a test file, so create the corresponding libtest structures.
812-
debug!(%file_path, "found test file");
813-
814-
// Record the stem of the test file, to check for overlaps later.
815-
let rel_test_path = relative_dir_path.join(file_path.file_stem().unwrap());
816-
collector.found_path_stems.insert(rel_test_path);
817-
818-
let paths =
819-
TestPaths { file: file_path, relative_dir: relative_dir_path.to_path_buf() };
820-
make_test(cx, collector, &paths);
821-
} else if file_path.is_dir() {
822-
// Recurse to find more tests in a subdirectory.
823-
let relative_file_path = relative_dir_path.join(file_name);
824-
if file_name != "auxiliary" {
825-
debug!(%file_path, "found directory");
826-
collect_tests_from_dir(cx, collector, &file_path, &relative_file_path)?;
813+
fs::read_dir(dir.as_std_path())?
814+
.par_bridge()
815+
.map(|file| {
816+
let mut collector = TestCollector::new();
817+
let file = file?;
818+
let file_path = Utf8PathBuf::try_from(file.path()).unwrap();
819+
let file_name = file_path.file_name().unwrap();
820+
821+
if is_test(file_name)
822+
&& (!cx.config.only_modified || cx.modified_tests.contains(&file_path))
823+
{
824+
// We found a test file, so create the corresponding libtest structures.
825+
debug!(%file_path, "found test file");
826+
827+
// Record the stem of the test file, to check for overlaps later.
828+
let rel_test_path = relative_dir_path.join(file_path.file_stem().unwrap());
829+
collector.found_path_stems.insert(rel_test_path);
830+
831+
let paths =
832+
TestPaths { file: file_path, relative_dir: relative_dir_path.to_path_buf() };
833+
make_test(cx, &mut collector, &paths);
834+
} else if file_path.is_dir() {
835+
// Recurse to find more tests in a subdirectory.
836+
let relative_file_path = relative_dir_path.join(file_name);
837+
if file_name != "auxiliary" {
838+
debug!(%file_path, "found directory");
839+
collector.merge(collect_tests_from_dir(cx, &file_path, &relative_file_path)?);
840+
}
841+
} else {
842+
debug!(%file_path, "found other file/directory");
827843
}
828-
} else {
829-
debug!(%file_path, "found other file/directory");
830-
}
831-
}
832-
Ok(())
844+
Ok(collector)
845+
})
846+
.reduce(
847+
|| Ok(TestCollector::new()),
848+
|a, b| {
849+
let mut a = a?;
850+
a.merge(b?);
851+
Ok(a)
852+
},
853+
)
833854
}
834855

835856
/// Returns true if `file_name` looks like a proper test file name.

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /