Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 5a3178f

Browse files
Auto merge of #144483 - scottmcm:early-inline, r=<try>
EXPERIMENT: Use the force-inline pass for more stuff
2 parents f32b232 + e9fcbb2 commit 5a3178f

30 files changed

+670
-49
lines changed

‎compiler/rustc_attr_data_structures/src/attributes.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,20 @@ pub enum InlineAttr {
2121
attr_span: Span,
2222
reason: Option<Symbol>,
2323
},
24+
/// `#[rustc_early_inline]` will always inline calls to a known impl in MIR.
25+
///
26+
/// You can think of this as either
27+
/// - Force, but without the "do not codegen as a function ever" restriction.
28+
/// - Always, but only for MIR.
29+
Early,
2430
}
2531

2632
impl InlineAttr {
27-
pub fn always(&self) -> bool {
33+
pub fn always_in_codegen(&self) -> bool {
2834
match self {
29-
InlineAttr::Always | InlineAttr::Force { .. } => true,
30-
InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
35+
InlineAttr::Always => true,
36+
InlineAttr::None | InlineAttr::Hint | InlineAttr::Early | InlineAttr::Never => false,
37+
InlineAttr::Force { .. } => panic!("Shouldn't be codegen'ing {self:?}"),
3138
}
3239
}
3340
}
@@ -349,7 +356,7 @@ pub enum AttributeKind {
349356
reason: Option<Symbol>,
350357
},
351358

352-
/// Represents `#[inline]` and `#[rustc_force_inline]`.
359+
/// Represents `#[inline]` and `#[rustc_force_inline]` and `#[rustc_early_inline]`.
353360
Inline(InlineAttr, Span),
354361

355362
/// Represents `#[link_name]`.

‎compiler/rustc_attr_parsing/src/attributes/inline.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,16 @@ impl<S: Stage> SingleAttributeParser<S> for RustcForceInlineParser {
9393
))
9494
}
9595
}
96+
97+
pub(crate) struct RustcEarlyInlineParser;
98+
99+
impl<S: Stage> SingleAttributeParser<S> for RustcEarlyInlineParser {
100+
const PATH: &'static [Symbol] = &[sym::rustc_early_inline];
101+
const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
102+
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
103+
const TEMPLATE: AttributeTemplate = template!(Word);
104+
105+
fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
106+
Some(AttributeKind::Inline(InlineAttr::Early, cx.attr_span))
107+
}
108+
}

‎compiler/rustc_attr_parsing/src/context.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use crate::attributes::codegen_attrs::{
2424
use crate::attributes::confusables::ConfusablesParser;
2525
use crate::attributes::deprecation::DeprecationParser;
2626
use crate::attributes::dummy::DummyParser;
27-
use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
27+
use crate::attributes::inline::{InlineParser, RustcEarlyInlineParser,RustcForceInlineParser};
2828
use crate::attributes::link_attrs::{
2929
ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser,
3030
LinkSectionParser, StdInternalSymbolParser,
@@ -154,6 +154,7 @@ attribute_parsers!(
154154
Single<MustUseParser>,
155155
Single<OptimizeParser>,
156156
Single<PathAttributeParser>,
157+
Single<RustcEarlyInlineParser>,
157158
Single<RustcForceInlineParser>,
158159
Single<RustcLayoutScalarValidRangeEnd>,
159160
Single<RustcLayoutScalarValidRangeStart>,

‎compiler/rustc_codegen_gcc/src/attributes.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ fn inline_attr<'gcc, 'tcx>(
6767
Some(FnAttribute::AlwaysInline)
6868
}
6969
}
70-
InlineAttr::Hint => Some(FnAttribute::Inline),
70+
InlineAttr::Hint | InlineAttr::Early=> Some(FnAttribute::Inline),
7171
InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
7272
InlineAttr::Never => {
7373
if cx.sess().target.arch != "amdgpu" {

‎compiler/rustc_codegen_llvm/src/attributes.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
5252
return Some(AttributeKind::NoInline.create_attr(cx.llcx));
5353
}
5454
match inline {
55-
InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
55+
InlineAttr::Hint | InlineAttr::Early => {
56+
Some(AttributeKind::InlineHint.create_attr(cx.llcx))
57+
}
5658
InlineAttr::Always | InlineAttr::Force { .. } => {
5759
Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
5860
}

‎compiler/rustc_codegen_ssa/src/codegen_attrs.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
452452
}
453453

454454
if !codegen_fn_attrs.no_sanitize.is_empty()
455-
&& codegen_fn_attrs.inline.always()
455+
&& codegen_fn_attrs.inline.always_in_codegen()
456456
&& let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span)
457457
{
458458
let hir_id = tcx.local_def_id_to_hir_id(did);

‎compiler/rustc_feature/src/builtin_attrs.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1123,6 +1123,11 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
11231123
rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
11241124
"`#[rustc_no_mir_inline]` prevents the MIR inliner from inlining a function while not affecting codegen"
11251125
),
1126+
rustc_attr!(
1127+
rustc_early_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
1128+
"`#[rustc_early_inline]` inlines non-generic calls to trivial methods even in debug, \
1129+
while still allowing them to be codegen'd for generic calls"
1130+
),
11261131
rustc_attr!(
11271132
rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
11281133
"`#[rustc_force_inline]` forces a free function to be inlined"

‎compiler/rustc_middle/src/mir/mono.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ impl<'tcx> MonoItem<'tcx> {
205205

206206
// To ensure that #[inline(always)] can be inlined as much as possible, especially in unoptimized
207207
// builds, we always select LocalCopy.
208-
if codegen_fn_attrs.inline.always() {
208+
if codegen_fn_attrs.inline.always_in_codegen() {
209209
return InstantiationMode::LocalCopy;
210210
}
211211

‎compiler/rustc_mir_transform/src/cross_crate_inline.rs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,10 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
4646
// #[inline(never)] to force code generation.
4747
match codegen_fn_attrs.inline {
4848
InlineAttr::Never => return false,
49-
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
50-
_ => {}
49+
InlineAttr::Hint | InlineAttr::Always | InlineAttr::Early | InlineAttr::Force { .. } => {
50+
return true;
51+
}
52+
InlineAttr::None => {}
5153
}
5254

5355
// If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until
@@ -77,7 +79,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
7779
// enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
7880
// which is less confusing than having to also enable -Copt-level=1.
7981
let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline, pm::Optimizations::Allowed)
80-
|| inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id());
82+
|| inline::ForceInline::applies_for_resolved_callee(tcx, def_id.to_def_id());
8183
if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run {
8284
return false;
8385
}

‎compiler/rustc_mir_transform/src/inline.rs

Lines changed: 42 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,13 @@ impl<'tcx> crate::MirPass<'tcx> for Inline {
7676
pub struct ForceInline;
7777

7878
impl ForceInline {
79-
pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
79+
pub fn needs_callgraph_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
8080
matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
8181
}
82+
83+
pub fn applies_for_resolved_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
84+
matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. } | InlineAttr::Early)
85+
}
8286
}
8387

8488
impl<'tcx> crate::MirPass<'tcx> for ForceInline {
@@ -117,7 +121,7 @@ trait Inliner<'tcx> {
117121
fn changed(self) -> bool;
118122

119123
/// Should inlining happen for a given callee?
120-
fn should_inline_for_callee(&self, def_id:DefId) -> bool;
124+
fn should_inline_for_callee(&self, instance_kind:&InstanceKind<'tcx>) -> bool;
121125

122126
fn check_codegen_attributes_extra(
123127
&self,
@@ -187,15 +191,22 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
187191
self.changed
188192
}
189193

190-
fn should_inline_for_callee(&self, def_id: DefId) -> bool {
191-
ForceInline::should_run_pass_for_callee(self.tcx(), def_id)
194+
fn should_inline_for_callee(&self, instance_kind: &InstanceKind<'tcx>) -> bool {
195+
if let InstanceKind::Item(def_id) = instance_kind
196+
&& let InlineAttr::Force { .. } | InlineAttr::Early =
197+
self.tcx().codegen_fn_attrs(def_id).inline
198+
{
199+
true
200+
} else {
201+
false
202+
}
192203
}
193204

194205
fn check_codegen_attributes_extra(
195206
&self,
196207
callee_attrs: &CodegenFnAttrs,
197208
) -> Result<(), &'static str> {
198-
debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
209+
debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. } | InlineAttr::Early);
199210
Ok(())
200211
}
201212

@@ -247,23 +258,26 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
247258

248259
fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
249260
let tcx = self.tcx();
250-
let InlineAttr::Force { attr_span, reason: justification } =
251-
tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
252-
else {
253-
bug!("called on item without required inlining");
254-
};
255-
256-
let call_span = callsite.source_info.span;
257-
tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
258-
call_span,
259-
attr_span,
260-
caller_span: tcx.def_span(self.def_id),
261-
caller: tcx.def_path_str(self.def_id),
262-
callee_span: tcx.def_span(callsite.callee.def_id()),
263-
callee: tcx.def_path_str(callsite.callee.def_id()),
264-
reason,
265-
justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
266-
});
261+
match tcx.codegen_fn_attrs(callsite.callee.def_id()).inline {
262+
InlineAttr::Early => {
263+
// Ok, we don't actually mind if this fails.
264+
}
265+
InlineAttr::Force { attr_span, reason: justification } => {
266+
let call_span = callsite.source_info.span;
267+
tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
268+
call_span,
269+
attr_span,
270+
caller_span: tcx.def_span(self.def_id),
271+
caller: tcx.def_path_str(self.def_id),
272+
callee_span: tcx.def_span(callsite.callee.def_id()),
273+
callee: tcx.def_path_str(callsite.callee.def_id()),
274+
reason,
275+
justification: justification
276+
.map(|sym| crate::errors::ForceInlineJustification { sym }),
277+
});
278+
}
279+
_ => bug!("called on item without required inlining"),
280+
}
267281
}
268282
}
269283

@@ -334,7 +348,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
334348
self.changed
335349
}
336350

337-
fn should_inline_for_callee(&self, _: DefId) -> bool {
351+
fn should_inline_for_callee(&self, _: &InstanceKind<'tcx>) -> bool {
338352
true
339353
}
340354

@@ -556,16 +570,16 @@ fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
556570
if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
557571
let func_ty = func.ty(caller_body, tcx);
558572
if let ty::FnDef(def_id, args) = *func_ty.kind() {
559-
if !inliner.should_inline_for_callee(def_id) {
560-
debug!("not enabled");
561-
return None;
562-
}
563-
564573
// To resolve an instance its args have to be fully normalized.
565574
let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?;
566575
let callee =
567576
Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?;
568577

578+
if !inliner.should_inline_for_callee(&callee.def) {
579+
debug!("not enabled");
580+
return None;
581+
}
582+
569583
if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
570584
return None;
571585
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /