diff --git a/crates/turbo-tasks-memory/src/aggregation/tests.rs b/crates/turbo-tasks-memory/src/aggregation/tests.rs
index bd6d5c2c821ad..12af07a9e7453 100644
--- a/crates/turbo-tasks-memory/src/aggregation/tests.rs
+++ b/crates/turbo-tasks-memory/src/aggregation/tests.rs
@@ -895,8 +895,7 @@ fn many_children() {
     }
     let children_duration = start.elapsed();
     println!("Children: {:?}", children_duration);
-    let mut number_of_slow_children = 0;
-    for j in 0..10 {
+    for j in 0.. {
         let start = Instant::now();
         for i in 0..CHILDREN {
             let node = Node::new(50000 + j * 10000 + i);
@@ -905,8 +904,12 @@ fn many_children() {
         }
         let dur = start.elapsed();
         println!("Children: {:?}", dur);
-        if dur > children_duration * 2 {
-            number_of_slow_children += 1;
+        let is_slow = dur > children_duration * 2;
+        if j > 10 && !is_slow {
+            break;
+        }
+        if j > 20 {
+            panic!("Adding children has become slower over time");
         }
     }
 
@@ -919,10 +922,6 @@ fn many_children() {
     }
     println!("Roots: {:?}", start.elapsed());
 
-    // Technically it should always be 0, but the performance of the environment
-    // might vary so we accept a few slow children
-    assert!(number_of_slow_children < 3);
-
     check_invariants(&ctx, roots.iter().cloned().map(NodeRef));
 
     // let root = NodeRef(roots[0].clone());