Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize integer pow by removing the exit branch #122884

Merged
merged 4 commits into from
Aug 14, 2024
Merged

Optimize integer pow by removing the exit branch #122884

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
df27dfa
Optimize integer pow by removing exit branch
mzabaluev Mar 22, 2024
1faa101
Explicitly unroll integer pow for small exponents
mzabaluev Jul 11, 2024
2f23534
Use is_val_statically_known to optimize pow
mzabaluev Jul 12, 2024
ac88b33
Revert to original loop for const pow exponents
mzabaluev Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions library/core/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@
#![feature(internal_impls_macro)]
#![feature(ip)]
#![feature(is_ascii_octdigit)]
#![feature(is_val_statically_known)]
#![feature(isqrt)]
#![feature(link_cfg)]
#![feature(offset_of_enum)]
Expand Down
139 changes: 98 additions & 41 deletions library/core/src/num/int_macros.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1495,18 +1495,17 @@ macro_rules! int_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return Some(acc);
}
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.checked_mul(base)
}

/// Strict exponentiation. Computes `self.pow(exp)`, panicking if
Expand Down Expand Up @@ -1546,18 +1545,17 @@ macro_rules! int_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = acc.strict_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.strict_mul(base);
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.strict_mul(base)
}

/// Returns the square root of the number, rounded down.
Expand Down Expand Up @@ -2174,26 +2172,57 @@ macro_rules! int_impl {
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[inline]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn wrapping_pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;

if intrinsics::is_val_statically_known(exp) {
// Unroll multiplications for small exponent values.
// This gives the optimizer a way to efficiently inline call sites
// for the most common use cases with constant exponents.
// Currently, LLVM is unable to unroll the loop below.
match exp {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rather than this special casing could we instead just have the original loop (which LLVM knows how to unroll) for the is_val_statically_known case and your new loop for the non-constant case?

And do the same for all the other pow functions.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have updated pow and wrapped_pow as suggested.
I'm not sure the extra complication is justified for the checked operations, but I guess the optimizer will have better opportunities with the original loop there as well. I will try to make a macro so that uniform code is used everywhere without repetition.

0 => return 1,
1 => return base,
2 => return base.wrapping_mul(base),
3 => {
let squared = base.wrapping_mul(base);
return squared.wrapping_mul(base);
}
4 => {
let squared = base.wrapping_mul(base);
return squared.wrapping_mul(squared);
}
5 => {
let squared = base.wrapping_mul(base);
return squared.wrapping_mul(squared).wrapping_mul(base);
}
6 => {
let cubed = base.wrapping_mul(base).wrapping_mul(base);
return cubed.wrapping_mul(cubed);
}
_ => {}
}
} else {
if exp == 0 {
return 1;
}
}
debug_assert!(exp != 0);

let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.wrapping_mul(base);
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
}

/// Calculates `self` + `rhs`
Expand Down Expand Up @@ -2687,9 +2716,14 @@ macro_rules! int_impl {
// Scratch space for storing results of overflowing_mul.
let mut r;

while exp > 1 {
loop {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
r.1 |= overflown;
return r;
}
acc = r.0;
overflown |= r.1;
}
Expand All @@ -2698,14 +2732,6 @@ macro_rules! int_impl {
base = r.0;
overflown |= r.1;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;
r
}

/// Raises self to the power of `exp`, using exponentiation by squaring.
Expand All @@ -2725,26 +2751,57 @@ macro_rules! int_impl {
without modifying the original"]
#[inline]
#[rustc_inherit_overflow_checks]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;

if intrinsics::is_val_statically_known(exp) {
// Unroll multiplications for small exponent values.
// This gives the optimizer a way to efficiently inline call sites
// for the most common use cases with constant exponents.
// Currently, LLVM is unable to unroll the loop below.
match exp {
0 => return 1,
1 => return base,
2 => return base * base,
3 => {
let squared = base * base;
return squared * base;
}
4 => {
let squared = base * base;
return squared * squared;
}
5 => {
let squared = base * base;
return squared * squared * base;
}
6 => {
let cubed = base * base * base;
return cubed * cubed;
}
_ => {}
}
} else {
if exp == 0 {
return 1;
}
}
debug_assert!(exp != 0);

let mut acc = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = acc * base;
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base * base;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
}

/// Returns the square root of the number, rounded down.
Expand Down
Loading
Loading