Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Add relnotes-api-list in-tree tool #143053

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pietroalbini wants to merge 5 commits into rust-lang:master
base: master
Choose a base branch
Loading
from pietroalbini:pa-relnotes-api-list
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions Cargo.lock
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3122,6 +3122,17 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"

[[package]]
name = "relnotes-api-list"
version = "0.1.0"
dependencies = [
"anyhow",
"rustdoc-json-types",
"serde",
"serde_json",
"tempfile",
]

[[package]]
name = "remote-test-client"
version = "0.1.0"
Expand Down
1 change: 1 addition & 0 deletions Cargo.toml
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ members = [
"src/tools/miri/cargo-miri",
"src/tools/miropt-test-tools",
"src/tools/opt-dist",
"src/tools/relnotes-api-list",
"src/tools/remote-test-client",
"src/tools/remote-test-server",
"src/tools/replace-version-placeholder",
Expand Down
6 changes: 6 additions & 0 deletions src/bootstrap/src/core/build_steps/check.rs
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,3 +556,9 @@ tool_check_step!(Compiletest {
allow_features: COMPILETEST_ALLOW_FEATURES,
default: false,
});

tool_check_step!(Linkchecker {
path: "src/tools/linkchecker",
mode: |_builder| Mode::ToolBootstrap,
default: false
});
59 changes: 59 additions & 0 deletions src/bootstrap/src/core/build_steps/dist.rs
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2641,3 +2641,62 @@
tarball.generate()
}
}

#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub struct RelnotesApiList {
pub host: TargetSelection,
}

impl Step for RelnotesApiList {
type Output = ();
const DEFAULT: bool = true;

fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
let default = run.builder.config.docs;
run.alias("relnotes-api-list").default_condition(default)
}

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(RelnotesApiList { host: run.target });
}

fn run(self, builder: &Builder<'_>) -> Self::Output {
let host = self.host;
let dest = builder.out.join("dist").join(format!("relnotes-api-list-{host}.json"));
builder.create_dir(dest.parent().unwrap());

// The HTML documentation for the standard library is needed to check all links generated by
// the tool are not broken.
builder.ensure(crate::core::build_steps::doc::Std::new(
builder.top_stage,
host,
DocumentationFormat::Html,
));

if std::env::var_os("EMILY_SKIP_DOC").is_none() {
// TODO: remove the condition

Check failure on line 2677 in src/bootstrap/src/core/build_steps/dist.rs

View workflow job for this annotation

GitHub Actions / PR - tidy 

TODO is used for tasks that should be done before merging a PR; If you want to leave a message in the codebase use FIXME

Check failure on line 2677 in src/bootstrap/src/core/build_steps/dist.rs

View workflow job for this annotation

GitHub Actions / PR - aarch64-gnu-llvm-19-2 

TODO is used for tasks that should be done before merging a PR; If you want to leave a message in the codebase use FIXME
builder.ensure(
crate::core::build_steps::doc::Std::new(
builder.top_stage,
host,
DocumentationFormat::Json,
)
// Crates containing symbols exported by any std crate:
.add_extra_crate("rustc-literal-escaper")
.add_extra_crate("std_detect"),
);
}

let linkchecker = builder.tool_exe(Tool::Linkchecker);

builder.info("Generating the API list for the release notes");
builder
.tool_cmd(Tool::RelnotesApiList)
.arg(builder.json_doc_out(host))
.arg(&dest)
.env("LINKCHECKER_PATH", linkchecker)
.env("STD_HTML_DOCS", builder.doc_out(self.host))
.run(builder);
builder.info(&format!("API list for the release notes available at {}", dest.display()));
}
}
12 changes: 10 additions & 2 deletions src/bootstrap/src/core/build_steps/doc.rs
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,11 +560,17 @@ pub struct Std {
pub target: TargetSelection,
pub format: DocumentationFormat,
crates: Vec<String>,
extra_crates: Vec<String>,
}

impl Std {
pub(crate) fn new(stage: u32, target: TargetSelection, format: DocumentationFormat) -> Self {
Std { stage, target, format, crates: vec![] }
Std { stage, target, format, crates: vec![], extra_crates: vec![] }
}

pub(crate) fn add_extra_crate(mut self, krate: &str) -> Self {
self.extra_crates.push(krate.to_string());
self
}
}

Expand Down Expand Up @@ -592,6 +598,7 @@ impl Step for Std {
DocumentationFormat::Html
},
crates,
extra_crates: vec![],
});
}

Expand All @@ -602,7 +609,7 @@ impl Step for Std {
fn run(self, builder: &Builder<'_>) {
let stage = self.stage;
let target = self.target;
let crates = if self.crates.is_empty() {
let mut crates = if self.crates.is_empty() {
builder
.in_tree_crates("sysroot", Some(target))
.iter()
Expand All @@ -611,6 +618,7 @@ impl Step for Std {
} else {
self.crates
};
crates.extend(self.extra_crates.iter().cloned());

let out = match self.format {
DocumentationFormat::Html => builder.doc_out(target),
Expand Down
1 change: 1 addition & 0 deletions src/bootstrap/src/core/build_steps/tool.rs
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,7 @@ bootstrap_tool!(
FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump";
OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"];
RunMakeSupport, "src/tools/run-make-support", "run_make_support", artifact_kind = ToolArtifactKind::Library;
RelnotesApiList, "src/tools/relnotes-api-list", "relnotes-api-list";
);

/// These are the submodules that are required for rustbook to work due to
Expand Down
7 changes: 5 additions & 2 deletions src/bootstrap/src/core/builder/mod.rs
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -986,7 +986,8 @@ impl<'a> Builder<'a> {
tool::LlvmBitcodeLinker,
tool::RustcPerf,
tool::WasmComponentLd,
tool::LldWrapper
tool::LldWrapper,
tool::RelnotesApiList,
),
Kind::Clippy => describe!(
clippy::Std,
Expand Down Expand Up @@ -1033,6 +1034,7 @@ impl<'a> Builder<'a> {
check::Compiletest,
check::FeaturesStatusDump,
check::CoverageDump,
check::Linkchecker,
// This has special staging logic, it may run on stage 1 while others run on stage 0.
// It takes quite some time to build stage 1, so put this at the end.
//
Expand Down Expand Up @@ -1162,7 +1164,8 @@ impl<'a> Builder<'a> {
dist::PlainSourceTarball,
dist::BuildManifest,
dist::ReproducibleArtifacts,
dist::Gcc
dist::Gcc,
dist::RelnotesApiList,
),
Kind::Install => describe!(
install::Docs,
Expand Down
2 changes: 1 addition & 1 deletion src/tools/linkchecker/Cargo.toml
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
name = "linkchecker"
version = "0.1.0"
edition = "2021"
edition = "2024"

[[bin]]
name = "linkchecker"
Expand Down
118 changes: 93 additions & 25 deletions src/tools/linkchecker/main.rs
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@
//! should catch the majority of "broken link" cases.
use std::cell::{Cell, RefCell};
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::io::ErrorKind;
use std::fs;
use std::iter::once;
use std::path::{Component, Path, PathBuf};
use std::rc::Rc;
use std::time::Instant;
use std::{env, fs};

use html5ever::tendril::ByteTendril;
use html5ever::tokenizer::{
Expand Down Expand Up @@ -110,10 +111,25 @@ macro_rules! t {
};
}

struct Cli {
docs: PathBuf,
link_targets_dirs: Vec<PathBuf>,
}

fn main() {
let docs = env::args_os().nth(1).expect("doc path should be first argument");
let docs = env::current_dir().unwrap().join(docs);
let mut checker = Checker { root: docs.clone(), cache: HashMap::new() };
let cli = match parse_cli() {
Ok(cli) => cli,
Err(err) => {
eprintln!("error: {err}");
usage_and_exit(1);
}
};

let mut checker = Checker {
root: cli.docs.clone(),
link_targets_dirs: cli.link_targets_dirs,
cache: HashMap::new(),
};
let mut report = Report {
errors: 0,
start: Instant::now(),
Expand All @@ -125,16 +141,58 @@ fn main() {
intra_doc_exceptions: 0,
has_broken_urls: false,
};
checker.walk(&docs, &mut report);
checker.walk(&cli.docs, &mut report);
report.report();
if report.errors != 0 {
println!("found some broken links");
std::process::exit(1);
}
}

fn parse_cli() -> Result<Cli, String> {
fn to_absolute_path(arg: &str) -> Result<PathBuf, String> {
std::path::absolute(arg).map_err(|e| format!("could not convert to absolute {arg}: {e}"))
}

let mut verbatim = false;
let mut docs = None;
let mut link_targets_dirs = Vec::new();

let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
if !verbatim && arg == "--" {
verbatim = true;
} else if !verbatim && (arg == "-h" || arg == "--help") {
usage_and_exit(0)
} else if !verbatim && arg == "--link-targets-dir" {
link_targets_dirs.push(to_absolute_path(
&args.next().ok_or("missing value for --link-targets-dir")?,
)?);
} else if !verbatim && let Some(value) = arg.strip_prefix("--link-targets-dir=") {
link_targets_dirs.push(to_absolute_path(value)?);
} else if !verbatim && arg.starts_with('-') {
return Err(format!("unknown flag: {arg}"));
} else if docs.is_none() {
docs = Some(arg);
} else {
return Err("too many positional arguments".into());
}
}

Ok(Cli {
docs: to_absolute_path(&docs.ok_or("missing first positional argument")?)?,
link_targets_dirs,
})
}

fn usage_and_exit(code: i32) -> ! {
eprintln!("usage: linkchecker PATH [--link-targets-dir=PATH ...]");
std::process::exit(code)
}

struct Checker {
root: PathBuf,
link_targets_dirs: Vec<PathBuf>,
cache: Cache,
}

Expand Down Expand Up @@ -420,37 +478,34 @@ impl Checker {

/// Load a file from disk, or from the cache if available.
fn load_file(&mut self, file: &Path, report: &mut Report) -> (String, &FileEntry) {
// https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
#[cfg(windows)]
const ERROR_INVALID_NAME: i32 = 123;

let pretty_path =
file.strip_prefix(&self.root).unwrap_or(file).to_str().unwrap().to_string();

let entry =
self.cache.entry(pretty_path.clone()).or_insert_with(|| match fs::metadata(file) {
for base in once(&self.root).chain(self.link_targets_dirs.iter()) {
let entry = self.cache.entry(pretty_path.clone());
if let Entry::Occupied(e) = &entry
&& !matches!(e.get(), FileEntry::Missing)
{
break;
}

let file = base.join(&pretty_path);
entry.insert_entry(match fs::metadata(&file) {
Ok(metadata) if metadata.is_dir() => FileEntry::Dir,
Ok(_) => {
if file.extension().and_then(|s| s.to_str()) != Some("html") {
FileEntry::OtherFile
} else {
report.html_files += 1;
load_html_file(file, report)
load_html_file(&file, report)
}
}
Err(e) if e.kind() == ErrorKind::NotFound => FileEntry::Missing,
Err(e) => {
// If a broken intra-doc link contains `::`, on windows, it will cause `ERROR_INVALID_NAME` rather than `NotFound`.
// Explicitly check for that so that the broken link can be allowed in `LINKCHECK_EXCEPTIONS`.
#[cfg(windows)]
if e.raw_os_error() == Some(ERROR_INVALID_NAME)
&& file.as_os_str().to_str().map_or(false, |s| s.contains("::"))
{
return FileEntry::Missing;
}
panic!("unexpected read error for {}: {}", file.display(), e);
}
Err(e) if is_not_found_error(&file, &e) => FileEntry::Missing,
Err(e) => panic!("unexpected read error for {}: {}", file.display(), e),
});
}

let entry = self.cache.get(&pretty_path).unwrap();
(pretty_path, entry)
}
}
Expand Down Expand Up @@ -629,3 +684,16 @@ fn parse_ids(ids: &mut HashSet<String>, file: &str, source: &str, report: &mut R
ids.insert(encoded);
}
}

fn is_not_found_error(path: &Path, error: &std::io::Error) -> bool {
// https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
const WINDOWS_ERROR_INVALID_NAME: i32 = 123;

error.kind() == std::io::ErrorKind::NotFound
// If a broken intra-doc link contains `::`, on windows, it will cause `ERROR_INVALID_NAME`
// rather than `NotFound`. Explicitly check for that so that the broken link can be allowed
// in `LINKCHECK_EXCEPTIONS`.
|| (cfg!(windows)
&& error.raw_os_error() == Some(WINDOWS_ERROR_INVALID_NAME)
&& path.as_os_str().to_str().map_or(false, |s| s.contains("::")))
}
11 changes: 11 additions & 0 deletions src/tools/relnotes-api-list/Cargo.toml
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[package]
name = "relnotes-api-list"
version = "0.1.0"
edition = "2024"

[dependencies]
anyhow = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
rustdoc-json-types = { path = "../../rustdoc-json-types" }
tempfile = "3.20.0"
17 changes: 17 additions & 0 deletions src/tools/relnotes-api-list/README.md
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# API list generator for the release notes

Rust's [release notes] include a "Stabilized APIs" section for each
release, listing all APIs that became stable each release. This tool supports
the creation of that section by generating a JSON file containing a concise
representation of the standard library API. The [relnotes tool] will then
compare the JSON files of two releases to generate the section.

The tool is executed by CI and produces the `relnotes-api-list-$target.json`
dist artifact. You can also run the tool locally with:

```
./x dist relnotes-api-list
```

[release notes]: https://doc.rust-lang.org/stable/releases.html
[relnotes tool]: https://github.com/rust-lang/relnotes
Loading
Loading

AltStyle によって変換されたページ (->オリジナル) /