Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit b2728d5

Browse files
committed
Auto merge of #135674 - scottmcm:assume-better, r=estebank
Update our range `assume`s to the format that LLVM prefers I found out in llvm/llvm-project#123278 (comment) that the way I started emitting the `assume`s in #109993 was suboptimal, and as seen in that LLVM issue the way we're doing it -- with two `assume`s sometimes -- can at times lead to CVP/SCCP not realize what's happening because one of them turns into a `ne` instead of conveying a range. So this updates how it's emitted from ``` assume( x >= LOW ); assume( x <= HIGH ); ``` or ``` // (for ranges that wrap the range) assume( (x <= LOW) | (x >= HIGH) ); ``` to ``` assume( (x - LOW) <= (HIGH - LOW) ); ``` so that we don't need multiple `icmp`s nor multiple `assume`s for a single value, and both wrappping and non-wrapping ranges emit the same shape. (And we don't bother emitting the subtraction if `LOW` is zero, since that's trivial for us to check too.)
2 parents c234b83 + 6fe8200 commit b2728d5

File tree

4 files changed

+114
-84
lines changed

4 files changed

+114
-84
lines changed

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs‎

Lines changed: 18 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -387,10 +387,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
387387
use abi::Primitive::*;
388388
imm = bx.from_immediate(imm);
389389

390-
// When scalars are passed by value, there's no metadata recording their
391-
// valid ranges. For example, `char`s are passed as just `i32`, with no
392-
// way for LLVM to know that they're 0x10FFFF at most. Thus we assume
393-
// the range of the input value too, not just the output range.
390+
// If we have a scalar, we must already know its range. Either
391+
//
392+
// 1) It's a parameter with `range` parameter metadata,
393+
// 2) It's something we `load`ed with `!range` metadata, or
394+
// 3) After a transmute we `assume`d the range (see below).
395+
//
396+
// That said, last time we tried removing this, it didn't actually help
397+
// the rustc-perf results, so might as well keep doing it
398+
// <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182>
394399
self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
395400

396401
imm = match (from_scalar.primitive(), to_scalar.primitive()) {
@@ -411,7 +416,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
411416
bx.bitcast(int_imm, to_backend_ty)
412417
}
413418
};
419+
420+
// This `assume` remains important for cases like (a conceptual)
421+
// transmute::<u32, NonZeroU32>(x) == 0
422+
// since it's never passed to something with parameter metadata (especially
423+
// after MIR inlining) so the only way to tell the backend about the
424+
// constraint that the `transmute` introduced is to `assume` it.
414425
self.assume_scalar_range(bx, imm, to_scalar, to_backend_ty);
426+
415427
imm = bx.to_immediate_scalar(imm, to_scalar);
416428
imm
417429
}
@@ -433,31 +445,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
433445
return;
434446
}
435447

436-
let abi::WrappingRange { start, end } = scalar.valid_range(self.cx);
437-
438-
if start <= end {
439-
if start > 0 {
440-
let low = bx.const_uint_big(backend_ty, start);
441-
let cmp = bx.icmp(IntPredicate::IntUGE, imm, low);
442-
bx.assume(cmp);
443-
}
444-
445-
let type_max = scalar.size(self.cx).unsigned_int_max();
446-
if end < type_max {
447-
let high = bx.const_uint_big(backend_ty, end);
448-
let cmp = bx.icmp(IntPredicate::IntULE, imm, high);
449-
bx.assume(cmp);
450-
}
451-
} else {
452-
let low = bx.const_uint_big(backend_ty, start);
453-
let cmp_low = bx.icmp(IntPredicate::IntUGE, imm, low);
454-
455-
let high = bx.const_uint_big(backend_ty, end);
456-
let cmp_high = bx.icmp(IntPredicate::IntULE, imm, high);
457-
458-
let or = bx.or(cmp_low, cmp_high);
459-
bx.assume(or);
460-
}
448+
let range = scalar.valid_range(self.cx);
449+
bx.assume_integer_range(imm, backend_ty, range);
461450
}
462451

463452
pub(crate) fn codegen_rvalue_unsized(

‎compiler/rustc_codegen_ssa/src/traits/builder.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,27 @@ pub trait BuilderMethods<'a, 'tcx>:
217217
dest: PlaceRef<'tcx, Self::Value>,
218218
);
219219

220+
/// Emits an `assume` that the integer value `imm` of type `ty` is contained in `range`.
221+
///
222+
/// This *always* emits the assumption, so you probably want to check the
223+
/// optimization level and `Scalar::is_always_valid` before calling it.
224+
fn assume_integer_range(&mut self, imm: Self::Value, ty: Self::Type, range: WrappingRange) {
225+
let WrappingRange { start, end } = range;
226+
227+
// Perhaps one day we'll be able to use assume operand bundles for this,
228+
// but for now this encoding with a single icmp+assume is best per
229+
// <https://github.com/llvm/llvm-project/issues/123278#issuecomment-2597440158>
230+
let shifted = if start == 0 {
231+
imm
232+
} else {
233+
let low = self.const_uint_big(ty, start);
234+
self.sub(imm, low)
235+
};
236+
let width = self.const_uint_big(ty, u128::wrapping_sub(end, start));
237+
let cmp = self.icmp(IntPredicate::IntULE, shifted, width);
238+
self.assume(cmp);
239+
}
240+
220241
fn range_metadata(&mut self, load: Self::Value, range: WrappingRange);
221242
fn nonnull_metadata(&mut self, load: Self::Value);
222243

‎tests/codegen/intrinsics/transmute-niched.rs‎

Lines changed: 67 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@ pub enum SmallEnum {
1717
// CHECK-LABEL: @check_to_enum(
1818
#[no_mangle]
1919
pub unsafe fn check_to_enum(x: i8) -> SmallEnum {
20-
// OPT: %0 = icmp uge i8 %x, 10
21-
// OPT: call void @llvm.assume(i1 %0)
22-
// OPT: %1 = icmp ule i8 %x, 12
20+
// CHECK-NOT: icmp
21+
// CHECK-NOT: assume
22+
// OPT: %0 = sub i8 %x, 10
23+
// OPT: %1 = icmp ule i8 %0, 2
2324
// OPT: call void @llvm.assume(i1 %1)
24-
// DBG-NOT: icmp
25-
// DBG-NOT: assume
25+
// CHECK-NOT: icmp
26+
// CHECK-NOT: assume
2627
// CHECK: ret i8 %x
2728

2829
transmute(x)
@@ -31,12 +32,13 @@ pub unsafe fn check_to_enum(x: i8) -> SmallEnum {
3132
// CHECK-LABEL: @check_from_enum(
3233
#[no_mangle]
3334
pub unsafe fn check_from_enum(x: SmallEnum) -> i8 {
34-
// OPT: %0 = icmp uge i8 %x, 10
35-
// OPT: call void @llvm.assume(i1 %0)
36-
// OPT: %1 = icmp ule i8 %x, 12
35+
// CHECK-NOT: icmp
36+
// CHECK-NOT: assume
37+
// OPT: %0 = sub i8 %x, 10
38+
// OPT: %1 = icmp ule i8 %0, 2
3739
// OPT: call void @llvm.assume(i1 %1)
38-
// DBG-NOT: icmp
39-
// DBG-NOT: assume
40+
// CHECK-NOT: icmp
41+
// CHECK-NOT: assume
4042
// CHECK: ret i8 %x
4143

4244
transmute(x)
@@ -45,12 +47,13 @@ pub unsafe fn check_from_enum(x: SmallEnum) -> i8 {
4547
// CHECK-LABEL: @check_to_ordering(
4648
#[no_mangle]
4749
pub unsafe fn check_to_ordering(x: u8) -> std::cmp::Ordering {
48-
// OPT: %0 = icmp uge i8 %x, -1
49-
// OPT: %1 = icmp ule i8 %x, 1
50-
// OPT: %2 = or i1 %0, %1
51-
// OPT: call void @llvm.assume(i1 %2)
52-
// DBG-NOT: icmp
53-
// DBG-NOT: assume
50+
// CHECK-NOT: icmp
51+
// CHECK-NOT: assume
52+
// OPT: %0 = sub i8 %x, -1
53+
// OPT: %1 = icmp ule i8 %0, 2
54+
// OPT: call void @llvm.assume(i1 %1)
55+
// CHECK-NOT: icmp
56+
// CHECK-NOT: assume
5457
// CHECK: ret i8 %x
5558

5659
transmute(x)
@@ -59,12 +62,13 @@ pub unsafe fn check_to_ordering(x: u8) -> std::cmp::Ordering {
5962
// CHECK-LABEL: @check_from_ordering(
6063
#[no_mangle]
6164
pub unsafe fn check_from_ordering(x: std::cmp::Ordering) -> u8 {
62-
// OPT: %0 = icmp uge i8 %x, -1
63-
// OPT: %1 = icmp ule i8 %x, 1
64-
// OPT: %2 = or i1 %0, %1
65-
// OPT: call void @llvm.assume(i1 %2)
66-
// DBG-NOT: icmp
67-
// DBG-NOT: assume
65+
// CHECK-NOT: icmp
66+
// CHECK-NOT: assume
67+
// OPT: %0 = sub i8 %x, -1
68+
// OPT: %1 = icmp ule i8 %0, 2
69+
// OPT: call void @llvm.assume(i1 %1)
70+
// CHECK-NOT: icmp
71+
// CHECK-NOT: assume
6872
// CHECK: ret i8 %x
6973

7074
transmute(x)
@@ -98,14 +102,15 @@ pub enum Minus100ToPlus100 {
98102
// CHECK-LABEL: @check_enum_from_char(
99103
#[no_mangle]
100104
pub unsafe fn check_enum_from_char(x: char) -> Minus100ToPlus100 {
105+
// CHECK-NOT: icmp
106+
// CHECK-NOT: assume
101107
// OPT: %0 = icmp ule i32 %x, 1114111
102108
// OPT: call void @llvm.assume(i1 %0)
103-
// OPT: %1 = icmp uge i32 %x, -100
104-
// OPT: %2 = icmp ule i32 %x, 100
105-
// OPT: %3 = or i1 %1, %2
106-
// OPT: call void @llvm.assume(i1 %3)
107-
// DBG-NOT: icmp
108-
// DBG-NOT: assume
109+
// OPT: %1 = sub i32 %x, -100
110+
// OPT: %2 = icmp ule i32 %1, 200
111+
// OPT: call void @llvm.assume(i1 %2)
112+
// CHECK-NOT: icmp
113+
// CHECK-NOT: assume
109114
// CHECK: ret i32 %x
110115

111116
transmute(x)
@@ -114,14 +119,15 @@ pub unsafe fn check_enum_from_char(x: char) -> Minus100ToPlus100 {
114119
// CHECK-LABEL: @check_enum_to_char(
115120
#[no_mangle]
116121
pub unsafe fn check_enum_to_char(x: Minus100ToPlus100) -> char {
117-
// OPT: %0 = icmp uge i32 %x, -100
118-
// OPT: %1 = icmp ule i32 %x, 100
119-
// OPT: %2 = or i1 %0, %1
122+
// CHECK-NOT: icmp
123+
// CHECK-NOT: assume
124+
// OPT: %0 = sub i32 %x, -100
125+
// OPT: %1 = icmp ule i32 %0, 200
126+
// OPT: call void @llvm.assume(i1 %1)
127+
// OPT: %2 = icmp ule i32 %x, 1114111
120128
// OPT: call void @llvm.assume(i1 %2)
121-
// OPT: %3 = icmp ule i32 %x, 1114111
122-
// OPT: call void @llvm.assume(i1 %3)
123-
// DBG-NOT: icmp
124-
// DBG-NOT: assume
129+
// CHECK-NOT: icmp
130+
// CHECK-NOT: assume
125131
// CHECK: ret i32 %x
126132

127133
transmute(x)
@@ -130,16 +136,20 @@ pub unsafe fn check_enum_to_char(x: Minus100ToPlus100) -> char {
130136
// CHECK-LABEL: @check_swap_pair(
131137
#[no_mangle]
132138
pub unsafe fn check_swap_pair(x: (char, NonZero<u32>)) -> (NonZero<u32>, char) {
139+
// CHECK-NOT: icmp
140+
// CHECK-NOT: assume
133141
// OPT: %0 = icmp ule i32 %x.0, 1114111
134142
// OPT: call void @llvm.assume(i1 %0)
135-
// OPT: %1 = icmp uge i32 %x.0, 1
136-
// OPT: call void @llvm.assume(i1 %1)
137-
// OPT: %2 = icmp uge i32 %x.1, 1
143+
// OPT: %1 = sub i32 %x.0, 1
144+
// OPT: %2 = icmp ule i32 %1, -2
138145
// OPT: call void @llvm.assume(i1 %2)
139-
// OPT: %3 = icmp ule i32 %x.1, 1114111
140-
// OPT: call void @llvm.assume(i1 %3)
141-
// DBG-NOT: icmp
142-
// DBG-NOT: assume
146+
// OPT: %3 = sub i32 %x.1, 1
147+
// OPT: %4 = icmp ule i32 %3, -2
148+
// OPT: call void @llvm.assume(i1 %4)
149+
// OPT: %5 = icmp ule i32 %x.1, 1114111
150+
// OPT: call void @llvm.assume(i1 %5)
151+
// CHECK-NOT: icmp
152+
// CHECK-NOT: assume
143153
// CHECK: %[[P1:.+]] = insertvalue { i32, i32 } poison, i32 %x.0, 0
144154
// CHECK: %[[P2:.+]] = insertvalue { i32, i32 } %[[P1]], i32 %x.1, 1
145155
// CHECK: ret { i32, i32 } %[[P2]]
@@ -150,14 +160,15 @@ pub unsafe fn check_swap_pair(x: (char, NonZero<u32>)) -> (NonZero<u32>, char) {
150160
// CHECK-LABEL: @check_bool_from_ordering(
151161
#[no_mangle]
152162
pub unsafe fn check_bool_from_ordering(x: std::cmp::Ordering) -> bool {
153-
// OPT: %0 = icmp uge i8 %x, -1
154-
// OPT: %1 = icmp ule i8 %x, 1
155-
// OPT: %2 = or i1 %0, %1
163+
// CHECK-NOT: icmp
164+
// CHECK-NOT: assume
165+
// OPT: %0 = sub i8 %x, -1
166+
// OPT: %1 = icmp ule i8 %0, 2
167+
// OPT: call void @llvm.assume(i1 %1)
168+
// OPT: %2 = icmp ule i8 %x, 1
156169
// OPT: call void @llvm.assume(i1 %2)
157-
// OPT: %3 = icmp ule i8 %x, 1
158-
// OPT: call void @llvm.assume(i1 %3)
159-
// DBG-NOT: icmp
160-
// DBG-NOT: assume
170+
// CHECK-NOT: icmp
171+
// CHECK-NOT: assume
161172
// CHECK: %[[R:.+]] = trunc i8 %x to i1
162173
// CHECK: ret i1 %[[R]]
163174

@@ -168,14 +179,15 @@ pub unsafe fn check_bool_from_ordering(x: std::cmp::Ordering) -> bool {
168179
#[no_mangle]
169180
pub unsafe fn check_bool_to_ordering(x: bool) -> std::cmp::Ordering {
170181
// CHECK: %_0 = zext i1 %x to i8
182+
// CHECK-NOT: icmp
183+
// CHECK-NOT: assume
171184
// OPT: %0 = icmp ule i8 %_0, 1
172185
// OPT: call void @llvm.assume(i1 %0)
173-
// OPT: %1 = icmp uge i8 %_0, -1
174-
// OPT: %2 = icmp ule i8 %_0, 1
175-
// OPT: %3 = or i1 %1, %2
176-
// OPT: call void @llvm.assume(i1 %3)
177-
// DBG-NOT: icmp
178-
// DBG-NOT: assume
186+
// OPT: %1 = sub i8 %_0, -1
187+
// OPT: %2 = icmp ule i8 %1, 2
188+
// OPT: call void @llvm.assume(i1 %2)
189+
// CHECK-NOT: icmp
190+
// CHECK-NOT: assume
179191
// CHECK: ret i8 %_0
180192

181193
transmute(x)

‎tests/codegen/transmute-optimized.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,11 @@ pub fn char_is_negative(c: char) -> bool {
110110
let x: i32 = unsafe { std::mem::transmute(c) };
111111
x < 0
112112
}
113+
114+
// CHECK-LABEL: i1 @transmute_to_char_is_negative(i32
115+
#[no_mangle]
116+
pub fn transmute_to_char_is_negative(x: i32) -> bool {
117+
// CHECK: ret i1 false
118+
let _c: char = unsafe { std::mem::transmute(x) };
119+
x < 0
120+
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /