From 1b9e556a86c544ab8dc45abfb5f6b787fc5655f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20FIDRY?= Date: Thu, 1 Aug 2024 22:15:18 +0200 Subject: [PATCH 1/4] feat: Adjust the load limit --- src/CpuCoreCounter.php | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/CpuCoreCounter.php b/src/CpuCoreCounter.php index 3fe3182..fc2ce87 100644 --- a/src/CpuCoreCounter.php +++ b/src/CpuCoreCounter.php @@ -43,6 +43,7 @@ public function __construct(?array $finders = null) } /** +<<<<<<< Updated upstream * @param positive-int|0 $reservedCpus Number of CPUs to reserve. This is useful when you want * to reserve some CPUs for other processes. If the main * process is going to be busy still, you may want to set @@ -55,16 +56,27 @@ public function __construct(?array $finders = null) * retrieved using `sys_getloadavg()` to check the load * of the system in the past minute. Should be a positive * float. +======= + * @param positive-int|0 $reservedCpus + * @param positive-int $limit + * @param float $loadLimit Element of [0., 1.]. Limits the number of CPUs based on the system load + * average. Set it to null or 1. to disable the check, it otherwise + * will adjust the number of CPUs based on the system load average. For example if 3 cores out of 10 are busy and the load limit is set to 50%, only 2 cores will be available for parallelisation. + * @param float $systemLoadAverage The system load average. If not provided, it will be + * retrieved using `sys_getloadavg()` to check the load + * of the system in the past minute. Should be a positive + * float. +>>>>>>> Stashed changes * * @see https://php.net/manual/en/function.sys-getloadavg.php */ public function getAvailableForParallelisation( int $reservedCpus = 0, ?int $limit = null, - ?float $loadLimitPerCore = .9, + ?float $loadLimit = .9, ?float $systemLoadAverage = null ): ParallelisationResult { - self::checkLoadLimitPerCore($loadLimitPerCore); + self::checkLoadLimitPerCore($loadLimit); self::checkSystemLoadAverage($systemLoadAverage); $correctedLimit = null === $limit @@ -81,7 +93,7 @@ public function getAvailableForParallelisation( $systemLoadAveragePerCore = $correctedSystemLoadAverage / $availableCpus; // Adjust available CPUs based on current load - if (null !== $loadLimitPerCore && $systemLoadAveragePerCore > $loadLimitPerCore) { + if (null !== $loadLimit && $systemLoadAveragePerCore > $loadLimit) { $adjustedCpus = max( 1, (1 - $systemLoadAveragePerCore) * $availableCpus @@ -96,7 +108,7 @@ public function getAvailableForParallelisation( return new ParallelisationResult( $reservedCpus, $limit, - $loadLimitPerCore, + $loadLimit, $systemLoadAverage, $correctedLimit, $correctedSystemLoadAverage, From a0365af8e492caab62307d754078945ba08bd11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20FIDRY?= Date: Fri, 2 Aug 2024 10:52:20 +0200 Subject: [PATCH 2/4] WIP --- src/CpuCoreCounter.php | 76 ++++++++++++++++------------------- src/ParallelisationResult.php | 2 +- tests/CpuCoreCounterTest.php | 22 ++++++++-- 3 files changed, 54 insertions(+), 46 deletions(-) diff --git a/src/CpuCoreCounter.php b/src/CpuCoreCounter.php index fc2ce87..de25eba 100644 --- a/src/CpuCoreCounter.php +++ b/src/CpuCoreCounter.php @@ -43,77 +43,71 @@ public function __construct(?array $finders = null) } /** -<<<<<<< Updated upstream - * @param positive-int|0 $reservedCpus Number of CPUs to reserve. This is useful when you want - * to reserve some CPUs for other processes. If the main - * process is going to be busy still, you may want to set - * this value to 1. - * - * @param positive-int $limit - * @param float $loadLimitPerCore Limits the number of CPUs based on the system load - * average per core in a range of [0., 1.]. - * @param float $systemLoadAverage The system load average. If not provided, it will be - * retrieved using `sys_getloadavg()` to check the load - * of the system in the past minute. Should be a positive - * float. -======= - * @param positive-int|0 $reservedCpus - * @param positive-int $limit - * @param float $loadLimit Element of [0., 1.]. Limits the number of CPUs based on the system load - * average. Set it to null or 1. to disable the check, it otherwise - * will adjust the number of CPUs based on the system load average. For example if 3 cores out of 10 are busy and the load limit is set to 50%, only 2 cores will be available for parallelisation. - * @param float $systemLoadAverage The system load average. If not provided, it will be + * @param positive-int|0 $reservedCpus Number of CPUs to reserve. This is useful when you want + * to reserve some CPUs for other processes. If the main + * process is going to be busy still, you may want to set + * this value to 1. + * @param float|null $systemLoadLimit Element of [0., 1.]. Limits the number of CPUs based + * on the system load average. Set it to null or 1. to + * disable the check, it otherwise will adjust the number + * of CPUs based on the system load average. For example + * if 3 cores out of 10 are busy and the load limit is + * set to 50%, only 2 cores will be available for + * parallelisation. The reserved cores are also taken + * into consideration, i.e. if .7 is passed, it will + * consider the max system load limit should not be + * higher than 70% for the system cores minus the reserved + * ones. + * @param float $systemLoadAverage The system load average. If not provided, it will be * retrieved using `sys_getloadavg()` to check the load * of the system in the past minute. Should be a positive * float. ->>>>>>> Stashed changes * * @see https://php.net/manual/en/function.sys-getloadavg.php */ public function getAvailableForParallelisation( int $reservedCpus = 0, ?int $limit = null, - ?float $loadLimit = .9, + ?float $systemLoadLimit = .9, ?float $systemLoadAverage = null ): ParallelisationResult { - self::checkLoadLimitPerCore($loadLimit); + self::checkLoadLimitPerCore($systemLoadLimit); self::checkSystemLoadAverage($systemLoadAverage); $correctedLimit = null === $limit ? self::getKubernetesLimit() : $limit; - $totalCoresCount = $this->getCountWithFallback(1); - - $availableCpus = max(1, $totalCoresCount - $reservedCpus); - - $correctedSystemLoadAverage = null === $systemLoadAverage - ? sys_getloadavg()[0] ?? 0. - : $systemLoadAverage; - $systemLoadAveragePerCore = $correctedSystemLoadAverage / $availableCpus; + $totalCoreCount = $this->getCountWithFallback(1); + $availableCores = max(1, $totalCoreCount - $reservedCpus); // Adjust available CPUs based on current load - if (null !== $loadLimit && $systemLoadAveragePerCore > $loadLimit) { - $adjustedCpus = max( + if (null !== $systemLoadLimit) { + $correctedSystemLoadAverage = null === $systemLoadAverage + ? sys_getloadavg()[0] ?? 0. + : $systemLoadAverage; + + $numberOfFreeCores = max( 1, - (1 - $systemLoadAveragePerCore) * $availableCpus + $systemLoadLimit * $totalCoreCount - $correctedSystemLoadAverage ); - $availableCpus = min($availableCpus, $adjustedCpus); + + $availableCores = min($availableCores, $numberOfFreeCores); } - if (null !== $correctedLimit && $availableCpus > $correctedLimit) { - $availableCpus = $correctedLimit; + if (null !== $correctedLimit && $availableCores > $correctedLimit) { + $availableCores = $correctedLimit; } return new ParallelisationResult( $reservedCpus, $limit, - $loadLimit, + $systemLoadLimit, $systemLoadAverage, $correctedLimit, - $correctedSystemLoadAverage, - $totalCoresCount, - (int) $availableCpus + $correctedSystemLoadAverage ?? $systemLoadAverage, + $totalCoreCount, + (int) $availableCores ); } diff --git a/src/ParallelisationResult.php b/src/ParallelisationResult.php index 1db52dc..eb3b404 100644 --- a/src/ParallelisationResult.php +++ b/src/ParallelisationResult.php @@ -70,7 +70,7 @@ public function __construct( ?float $passedLoadLimitPerCore, ?float $passedSystemLoadAverage, ?int $correctedLimit, - float $correctedSystemLoadAverage, + ?float $correctedSystemLoadAverage, int $totalCoresCount, int $availableCpus ) { diff --git a/tests/CpuCoreCounterTest.php b/tests/CpuCoreCounterTest.php index 97aa82c..bfe92d3 100644 --- a/tests/CpuCoreCounterTest.php +++ b/tests/CpuCoreCounterTest.php @@ -307,10 +307,10 @@ public static function availableCpuCoreProvider(): iterable return [ [$finder], [], + 0, null, null, - null, - null, + .0, 5, ]; })(); @@ -367,7 +367,7 @@ public static function availableCpuCoreProvider(): iterable null, .9, 6., - 10, + 3, ]; })(); @@ -386,7 +386,7 @@ public static function availableCpuCoreProvider(): iterable })(); yield 'CPU count found, the CPUs are being the limit set, but there is several CPUs available still' => (static function () { - $finder = new DummyCpuCoreFinder(11); + $finder = new DummyCpuCoreFinder(10); return [ [$finder], @@ -426,6 +426,20 @@ public static function availableCpuCoreProvider(): iterable 10, ]; })(); + + yield 'load limit doc example' => (static function () { + $finder = new DummyCpuCoreFinder(10); + + return [ + [$finder], + [], + 0, + null, + .5, + 3, + 2, + ]; + })(); } /** From f0ed9f74033aef4fdfa943cb7dc19d7cdc47edd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20FIDRY?= Date: Fri, 2 Aug 2024 12:01:36 +0200 Subject: [PATCH 3/4] rework --- src/CpuCoreCounter.php | 40 ++++++++++++++--------------- tests/AvailableCpuCoresScenario.php | 8 +++--- tests/CpuCoreCounterTest.php | 26 +++++++++++++------ 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/CpuCoreCounter.php b/src/CpuCoreCounter.php index de25eba..46d10a7 100644 --- a/src/CpuCoreCounter.php +++ b/src/CpuCoreCounter.php @@ -47,18 +47,18 @@ public function __construct(?array $finders = null) * to reserve some CPUs for other processes. If the main * process is going to be busy still, you may want to set * this value to 1. - * @param float|null $systemLoadLimit Element of [0., 1.]. Limits the number of CPUs based - * on the system load average. Set it to null or 1. to - * disable the check, it otherwise will adjust the number - * of CPUs based on the system load average. For example - * if 3 cores out of 10 are busy and the load limit is - * set to 50%, only 2 cores will be available for - * parallelisation. The reserved cores are also taken - * into consideration, i.e. if .7 is passed, it will - * consider the max system load limit should not be - * higher than 70% for the system cores minus the reserved - * ones. - * @param float $systemLoadAverage The system load average. If not provided, it will be + * @param float|null $loadLimit Element of [0., 1.]. Percentage representing the + * amount of cores that should be used among the available + * resources. For instance, if set to 0.7, it will use 70% + * of the available cores, i.e. if 1 core is reserved, 11 + * cores are available and 5 are busy, it will use 70% + * of (11-1-5)=5 cores, so 3 cores. Set this parameter to null + * to skip this check. Beware that 1 does not mean "no limit", + * but 100% of the _available_ resources, i.e. with the + * previous example, it will return 5 cores. How busy is + * the system is determined by the system load average + * (see $systemLoadAverage). + * @param float $systemLoadAverage The system load average. If not provided, it will be * retrieved using `sys_getloadavg()` to check the load * of the system in the past minute. Should be a positive * float. @@ -68,10 +68,10 @@ public function __construct(?array $finders = null) public function getAvailableForParallelisation( int $reservedCpus = 0, ?int $limit = null, - ?float $systemLoadLimit = .9, + ?float $loadLimit = null, ?float $systemLoadAverage = null ): ParallelisationResult { - self::checkLoadLimitPerCore($systemLoadLimit); + self::checkLoadLimit($loadLimit); self::checkSystemLoadAverage($systemLoadAverage); $correctedLimit = null === $limit @@ -82,17 +82,15 @@ public function getAvailableForParallelisation( $availableCores = max(1, $totalCoreCount - $reservedCpus); // Adjust available CPUs based on current load - if (null !== $systemLoadLimit) { + if (null !== $loadLimit) { $correctedSystemLoadAverage = null === $systemLoadAverage ? sys_getloadavg()[0] ?? 0. : $systemLoadAverage; - $numberOfFreeCores = max( + $availableCores = max( 1, - $systemLoadLimit * $totalCoreCount - $correctedSystemLoadAverage + $loadLimit * ($availableCores - $correctedSystemLoadAverage) ); - - $availableCores = min($availableCores, $numberOfFreeCores); } if (null !== $correctedLimit && $availableCores > $correctedLimit) { @@ -102,7 +100,7 @@ public function getAvailableForParallelisation( return new ParallelisationResult( $reservedCpus, $limit, - $systemLoadLimit, + $loadLimit, $systemLoadAverage, $correctedLimit, $correctedSystemLoadAverage ?? $systemLoadAverage, @@ -211,7 +209,7 @@ public static function getKubernetesLimit(): ?int return $finder->find(); } - private static function checkLoadLimitPerCore(?float $loadLimitPerCore): void + private static function checkLoadLimit(?float $loadLimitPerCore): void { if (null === $loadLimitPerCore) { return; diff --git a/tests/AvailableCpuCoresScenario.php b/tests/AvailableCpuCoresScenario.php index 1a5ef55..ba2309c 100644 --- a/tests/AvailableCpuCoresScenario.php +++ b/tests/AvailableCpuCoresScenario.php @@ -49,7 +49,7 @@ public function __construct( array $environmentVariables, int $reservedCpus, ?int $limit, - ?float $loadLimitPerCore, + ?float $loadLimit, ?float $systemLoadAverage, int $expected ) { @@ -57,7 +57,7 @@ public function __construct( $this->environmentVariables = $environmentVariables; $this->reservedCpus = $reservedCpus; $this->limit = $limit; - $this->loadLimitPerCore = $loadLimitPerCore; + $this->loadLimitPerCore = $loadLimit; $this->systemLoadAverage = $systemLoadAverage; $this->expected = $expected; } @@ -76,7 +76,7 @@ public static function create( array $environmentVariables, ?int $reservedCpus, ?int $limit, - ?float $loadLimitPerCore, + ?float $loadLimit, ?float $systemLoadAverage, int $expected ): array { @@ -90,7 +90,7 @@ public static function create( $environmentVariables, $reservedCpus ?? 0, $limit, - $loadLimitPerCore, + $loadLimit, $systemLoadAverage ?? 0., $expected ), diff --git a/tests/CpuCoreCounterTest.php b/tests/CpuCoreCounterTest.php index 5eb7ba4..8211dc8 100644 --- a/tests/CpuCoreCounterTest.php +++ b/tests/CpuCoreCounterTest.php @@ -299,16 +299,26 @@ public static function availableCpuCoreProvider(): iterable 1 ); - yield 'CPU count found, over half the cores are used' => AvailableCpuCoresScenario::create( + yield 'CPU count found, over half the cores are used and no limit is set' => AvailableCpuCoresScenario::create( 11, [], 1, null, - .9, + null, 6., 10 ); + yield 'CPU count found, over half the cores are used and a limit is set' => AvailableCpuCoresScenario::create( + 11, + [], + 1, + null, + 1., + 6., + 4 + ); + yield 'CPU count found, the CPUs are overloaded' => AvailableCpuCoresScenario::create( 11, [], @@ -319,24 +329,24 @@ public static function availableCpuCoreProvider(): iterable 1 ); - yield 'CPU count found, the CPUs are being the limit set, but there is several CPUs available still' => AvailableCpuCoresScenario::create( + yield 'CPU count found, the load limit is set, but there is several CPUs available still' => AvailableCpuCoresScenario::create( 11, [], 1, null, .5, 6., - 4 + 2 ); - yield 'CPU count found, the CPUs are at the limit of being overloaded' => AvailableCpuCoresScenario::create( + yield 'CPU count found, the CPUs are at completely overloaded' => AvailableCpuCoresScenario::create( 11, [], 1, null, - .9, - 9., - 10 + .5, + 11., + 1 ); yield 'CPU count found, the CPUs are overloaded but no load limit per CPU' => AvailableCpuCoresScenario::create( From c80af3a4e5f2814cb99efa97f0f918b4617e6f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20FIDRY?= Date: Fri, 2 Aug 2024 12:06:46 +0200 Subject: [PATCH 4/4] fix --- src/CpuCoreCounter.php | 3 ++- src/ParallelisationResult.php | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/CpuCoreCounter.php b/src/CpuCoreCounter.php index 46d10a7..a41532c 100644 --- a/src/CpuCoreCounter.php +++ b/src/CpuCoreCounter.php @@ -47,6 +47,7 @@ public function __construct(?array $finders = null) * to reserve some CPUs for other processes. If the main * process is going to be busy still, you may want to set * this value to 1. + * @param positive-int $limit * @param float|null $loadLimit Element of [0., 1.]. Percentage representing the * amount of cores that should be used among the available * resources. For instance, if set to 0.7, it will use 70% @@ -58,7 +59,7 @@ public function __construct(?array $finders = null) * previous example, it will return 5 cores. How busy is * the system is determined by the system load average * (see $systemLoadAverage). - * @param float $systemLoadAverage The system load average. If not provided, it will be + * @param float|null $systemLoadAverage The system load average. If not provided, it will be * retrieved using `sys_getloadavg()` to check the load * of the system in the past minute. Should be a positive * float. diff --git a/src/ParallelisationResult.php b/src/ParallelisationResult.php index eb3b404..36e12c5 100644 --- a/src/ParallelisationResult.php +++ b/src/ParallelisationResult.php @@ -31,7 +31,7 @@ final class ParallelisationResult /** * @var float|null */ - public $passedLoadLimitPerCore; + public $passedLoadLimit; /** * @var float|null @@ -44,7 +44,7 @@ final class ParallelisationResult public $correctedLimit; /** - * @var float + * @var float|null */ public $correctedSystemLoadAverage; @@ -67,7 +67,7 @@ final class ParallelisationResult public function __construct( int $passedReservedCpus, ?int $passedLimit, - ?float $passedLoadLimitPerCore, + ?float $passedLoadLimit, ?float $passedSystemLoadAverage, ?int $correctedLimit, ?float $correctedSystemLoadAverage, @@ -76,7 +76,7 @@ public function __construct( ) { $this->passedReservedCpus = $passedReservedCpus; $this->passedLimit = $passedLimit; - $this->passedLoadLimitPerCore = $passedLoadLimitPerCore; + $this->passedLoadLimit = $passedLoadLimit; $this->passedSystemLoadAverage = $passedSystemLoadAverage; $this->correctedLimit = $correctedLimit; $this->correctedSystemLoadAverage = $correctedSystemLoadAverage;