Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize integer pow by removing the exit branch #122884

Merged
merged 4 commits into from
Aug 14, 2024
Merged

Optimize integer pow by removing the exit branch #122884

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
df27dfa
Optimize integer pow by removing exit branch
mzabaluev Mar 22, 2024
1faa101
Explicitly unroll integer pow for small exponents
mzabaluev Jul 11, 2024
2f23534
Use is_val_statically_known to optimize pow
mzabaluev Jul 12, 2024
ac88b33
Revert to original loop for const pow exponents
mzabaluev Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions library/core/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@
#![feature(internal_impls_macro)]
#![feature(ip)]
#![feature(is_ascii_octdigit)]
#![feature(is_val_statically_known)]
#![feature(isqrt)]
#![feature(link_cfg)]
#![feature(offset_of_enum)]
Expand Down
118 changes: 75 additions & 43 deletions library/core/src/num/int_macros.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1495,18 +1495,17 @@ macro_rules! int_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return Some(acc);
}
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.checked_mul(base)
}

/// Strict exponentiation. Computes `self.pow(exp)`, panicking if
Expand Down Expand Up @@ -1546,18 +1545,17 @@ macro_rules! int_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = acc.strict_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.strict_mul(base);
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.strict_mul(base)
}

/// Returns the square root of the number, rounded down.
Expand Down Expand Up @@ -2174,26 +2172,44 @@ macro_rules! int_impl {
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[inline]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn wrapping_pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary.
acc.wrapping_mul(base)
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.wrapping_mul(base);
}
}
}

/// Calculates `self` + `rhs`
Expand Down Expand Up @@ -2687,9 +2703,14 @@ macro_rules! int_impl {
// Scratch space for storing results of overflowing_mul.
let mut r;

while exp > 1 {
loop {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
r.1 |= overflown;
return r;
}
acc = r.0;
overflown |= r.1;
}
Expand All @@ -2698,14 +2719,6 @@ macro_rules! int_impl {
base = r.0;
overflown |= r.1;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;
r
}

/// Raises self to the power of `exp`, using exponentiation by squaring.
Expand All @@ -2725,26 +2738,45 @@ macro_rules! int_impl {
without modifying the original"]
#[inline]
#[rustc_inherit_overflow_checks]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
}
exp /= 2;
base = base * base;
}
exp /= 2;
base = base * base;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc * base;
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base * base;
}
}
}

/// Returns the square root of the number, rounded down.
Expand Down
121 changes: 75 additions & 46 deletions library/core/src/num/uint_macros.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1534,20 +1534,17 @@ macro_rules! uint_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return Some(acc);
}
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.

acc.checked_mul(base)
}

/// Strict exponentiation. Computes `self.pow(exp)`, panicking if
Expand Down Expand Up @@ -1587,18 +1584,17 @@ macro_rules! uint_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = acc.strict_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.strict_mul(base);
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.strict_mul(base)
}

/// Saturating integer addition. Computes `self + rhs`, saturating at
Expand Down Expand Up @@ -2052,26 +2048,44 @@ macro_rules! uint_impl {
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[inline]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn wrapping_pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary.
acc.wrapping_mul(base)
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.wrapping_mul(base);
}
}
}

/// Calculates `self` + `rhs`
Expand Down Expand Up @@ -2516,9 +2530,14 @@ macro_rules! uint_impl {
// Scratch space for storing results of overflowing_mul.
let mut r;

while exp > 1 {
loop {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
r.1 |= overflown;
return r;
}
acc = r.0;
overflown |= r.1;
}
Expand All @@ -2527,15 +2546,6 @@ macro_rules! uint_impl {
base = r.0;
overflown |= r.1;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;

r
}

/// Raises self to the power of `exp`, using exponentiation by squaring.
Expand All @@ -2553,26 +2563,45 @@ macro_rules! uint_impl {
without modifying the original"]
#[inline]
#[rustc_inherit_overflow_checks]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
}
exp /= 2;
base = base * base;
}
exp /= 2;
base = base * base;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc * base;
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base * base;
}
}
}

/// Returns the square root of the number, rounded down.
Expand Down
Loading