Enable ParallelSuiteTest (#2687)

* Enable ParallelSuiteTest Presumably, test was disabled to get rid of distraction during development and was forgotten. Test was broken and commented out temporary back in 2011 in cc950e9 commit, named `Temp commit`. Commit was introduced in `AllDynamic` branch Test is green both on the base commit of AllDynamic branch (10a202a) as well as on branch merge point (b251fce) Done: - Add test: child suite should obey threadCount parameter - Move private stuff to bottom, add more details in error message * Fix: setup parallel option for all suites, including child * Fix populateSuiteGraph method: add parent nodes, too Before this, populateSuiteGraph method was ignoring any suite runner with child suites. This leads to parallel suites not being executed with `randomize suite` option enabled The method now add all nodes, including parents into the graph. * Adjust expected results for suitesShouldRunInParallel4 This test implied a requirement to exclude duplicating suite files from run. For example, for suite graph like: suite-parallel-1.xml suite-parallel-2.xml suite-parallel-2-1.xml suite-parallel-2-2.xml | | | | ---------- ---------| v v v suite-parallel-2-2-1.xml suite-parallel-2-1.xml suite-parallel-2-2.xml | | v suite-parallel-2-2-1.xml a test expected 5 suites and 5 threads. However, this test was excluded from TestNG unit tests since 2011, and broken somewhere later on the timeline. Dropping mentioned requirement (i.e. leaving things as it is now), since there's no complains on current behaviour by adjusting test expected results, to 8 expected suites and 8 threads. * review fixes: - revert suite order - gradle autostyle apply - set parallel mode recursively Co-authored-by: sankouski-dzmitry <[email protected]> Co-authored-by: Dzmitry Sankouski <[email protected]>
testng-team · Dec 12, 2021 · f264982 · f264982
1 parent 290532e
commit f264982
Show file tree

Hide file tree

Showing 9 changed files with 132 additions and 55 deletions.
diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java
@@ -344,12 +344,7 @@ private Collection<XmlSuite> parseSuite(String suitePath) {
   private Collection<XmlSuite> processCommandLineArgs(Collection<XmlSuite> allSuites) {
     Collection<XmlSuite> result = new ArrayList<>();
     for (XmlSuite s : allSuites) {
-      if (this.m_parallelMode != null) {
-        s.setParallel(this.m_parallelMode);
-      }
-      if (this.m_threadCount > 0) {
-        s.setThreadCount(this.m_threadCount);
-      }
+      processParallelModeCommandLineArgs(s);
       if (m_testNames == null) {
         result.add(s);
         continue;
@@ -366,6 +361,18 @@ private Collection<XmlSuite> processCommandLineArgs(Collection<XmlSuite> allSuit
     return result;
   }
 
+  private void processParallelModeCommandLineArgs(XmlSuite suite) {
+    if (this.m_parallelMode != null) {
+      suite.setParallel(this.m_parallelMode);
+    }
+    if (this.m_threadCount > 0) {
+      suite.setThreadCount(this.m_threadCount);
+    }
+    if (suite.getChildSuites() != null) {
+      suite.getChildSuites().forEach(this::processParallelModeCommandLineArgs);
+    }
+  }
+
   public void initializeSuitesAndJarFile() {
     // The IntelliJ plug-in might have invoked this method already so don't initialize suites twice.
     if (isSuiteInitialized) {
@@ -1256,9 +1263,8 @@ private void populateSuiteGraph(
       SuiteRunnerMap suiteRunnerMap,
       XmlSuite xmlSuite) {
     ISuite parentSuiteRunner = suiteRunnerMap.get(xmlSuite);
-    if (xmlSuite.getChildSuites().isEmpty()) {
-      suiteGraph.addNode(parentSuiteRunner);
-    } else {
+    suiteGraph.addNode(parentSuiteRunner);
+    if (!xmlSuite.getChildSuites().isEmpty()) {
       for (XmlSuite childSuite : xmlSuite.getChildSuites()) {
         suiteGraph.addEdge(0, parentSuiteRunner, suiteRunnerMap.get(childSuite));
         populateSuiteGraph(suiteGraph, suiteRunnerMap, childSuite);

diff --git a/testng-core/src/test/java/test/thread/ParallelSuiteTest.java b/testng-core/src/test/java/test/thread/ParallelSuiteTest.java
@@ -9,6 +9,7 @@
 import org.testng.TestListenerAdapter;
 import org.testng.TestNG;
 import org.testng.annotations.Test;
+import org.testng.xml.XmlSuite;
 import test.SimpleBaseTest;
 
 public class ParallelSuiteTest extends SimpleBaseTest {
@@ -21,12 +22,18 @@ public void suitesShouldRunInParallel1() {
         2,
         null,
         Arrays.asList(
-            getPathToResource("suite-parallel-1.xml"), getPathToResource("suite-parallel-2.xml")));
+            getPathToParallelResource("simple-suite-parallel-1.xml"),
+            getPathToParallelResource("simple-suite-parallel-2.xml")));
   }
 
   @Test
   public void suitesShouldRunInParallel2() {
-    runTest(5, 3, 3, null, Collections.singletonList(getPathToResource("suite-parallel-0.xml")));
+    runTest(
+        5,
+        3,
+        3,
+        null,
+        Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml")));
   }
 
   @Test(description = "Number of threads (2) is less than number of suites (3)")
@@ -35,54 +42,28 @@ public void suitesShouldRunInParallel3() {
     TestListenerAdapter tla = new TestListenerAdapter();
     TestNG tng = create();
     tng.setSuiteThreadPoolSize(SUITE_THREAD_POOL_SIZE);
-    tng.setTestSuites(Collections.singletonList(getPathToResource("suite-parallel-0.xml")));
+    tng.setTestSuites(
+        Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml")));
     tng.addListener((ITestNGListener) tla);
 
     BaseThreadTest.initThreadLog();
     tng.run(); // Shouldn't not deadlock
     Assert.assertEquals(BaseThreadTest.getThreadCount(), SUITE_THREAD_POOL_SIZE);
   }
 
-  private void runTest(
-      int suiteThreadPoolSize,
-      int expectedThreadCount,
-      int expectedSuiteCount,
-      Boolean randomizeSuites,
-      List<String> paths) {
-    TestListenerAdapter tla = new TestListenerAdapter();
-    TestNG tng = create();
-    tng.setSuiteThreadPoolSize(suiteThreadPoolSize);
-    tng.setTestSuites(paths);
-    tng.addListener((ITestNGListener) tla);
-    if (null != randomizeSuites) {
-      tng.setRandomizeSuites(randomizeSuites);
-    }
-
-    BaseThreadTest.initThreadLog();
-    tng.run();
-
-    Assert.assertEquals(
-        BaseThreadTest.getThreadCount(),
-        expectedThreadCount,
-        "Thread count expected:"
-            + expectedThreadCount
-            + " actual:"
-            + BaseThreadTest.getThreadCount());
-    Assert.assertEquals(BaseThreadTest.getSuitesMap().keySet().size(), expectedSuiteCount);
-  }
-
   @Test
   public void suitesShouldRunInParallel4() {
+    final int TOTAL_SUITE_COUNT_INCLUDING_DUPLICATES = 8;
     runTest(
         10,
-        5,
-        5,
+        TOTAL_SUITE_COUNT_INCLUDING_DUPLICATES,
+        TOTAL_SUITE_COUNT_INCLUDING_DUPLICATES,
         null,
         Arrays.asList(
-            getPathToResource("parallel-suites/suite-parallel-1.xml"),
-            getPathToResource("parallel-suites/suite-parallel-2.xml"),
-            getPathToResource("parallel-suites/suite-parallel-2-1.xml"),
-            getPathToResource("parallel-suites/suite-parallel-2-2.xml")));
+            getPathToParallelResource("suite-parallel-1.xml"),
+            getPathToParallelResource("suite-parallel-2.xml"),
+            getPathToParallelResource("suite-parallel-2-1.xml"),
+            getPathToParallelResource("suite-parallel-2-2.xml")));
   }
 
   @Test
@@ -92,7 +73,7 @@ public void suitesShouldRunInParallel5() {
         5,
         7,
         null,
-        Collections.singletonList(getPathToResource("parallel-suites/suite-parallel-0.xml")));
+        Collections.singletonList(getPathToParallelResource("suite-parallel-0.xml")));
   }
 
   @Test(description = "Number of threads (2) is less than level of suites (3)")
@@ -102,7 +83,7 @@ public void suitesShouldRunInParallel6() {
         2,
         7,
         null,
-        Collections.singletonList(getPathToResource("parallel-suites/suite-parallel-0.xml")));
+        Collections.singletonList(getPathToParallelResource("suite-parallel-0.xml")));
   }
 
   @Test(
@@ -112,7 +93,8 @@ public void suitesShouldRunInParallel6() {
   public void suitesShouldRunInOrder() {
     TestListenerAdapter tla = new TestListenerAdapter();
     TestNG tng = create();
-    tng.setTestSuites(Collections.singletonList(getPathToResource("suite-parallel-0.xml")));
+    tng.setTestSuites(
+        Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml")));
     tng.addListener((ITestNGListener) tla);
     BaseThreadTest.initThreadLog();
     tng.run();
@@ -134,6 +116,84 @@ public void suitesShouldRunInOrder() {
 
   @Test(description = "Number of threads (1) is less than number of levels of suites (2)")
   public void suitesShouldRun1() {
-    runTest(1, 1, 3, true, Collections.singletonList(getPathToResource("suite-parallel-0.xml")));
+    runTest(
+        1,
+        1,
+        3,
+        true,
+        Collections.singletonList(getPathToParallelResource("simple-suite-parallel-0.xml")));
+  }
+
+  @Test(description = "Child suite should obey threadCount parameter")
+  public void childSuiteObeyParentThreadCount() {
+    /* parent suite has no tests, so only child suite counts */
+    final int EXPECTED_SUITE_COUNT = 1;
+    runTest(
+        1,
+        2,
+        XmlSuite.ParallelMode.CLASSES,
+        2,
+        EXPECTED_SUITE_COUNT,
+        null,
+        Arrays.asList(getPathToParallelResource("inherit-thread-count-parent.yaml")));
+  }
+
+  private void runTest(
+      int suiteThreadPoolSize,
+      int expectedThreadCount,
+      int expectedSuiteCount,
+      Boolean randomizeSuites,
+      List<String> paths) {
+    runTest(
+        suiteThreadPoolSize,
+        null,
+        null,
+        expectedThreadCount,
+        expectedSuiteCount,
+        randomizeSuites,
+        paths);
+  }
+
+  private void runTest(
+      int suiteThreadPoolSize,
+      Integer threadCount,
+      XmlSuite.ParallelMode parallelMode,
+      int expectedThreadCount,
+      int expectedSuiteCount,
+      Boolean randomizeSuites,
+      List<String> paths) {
+    TestListenerAdapter tla = new TestListenerAdapter();
+    TestNG tng = create();
+    tng.setSuiteThreadPoolSize(suiteThreadPoolSize);
+    if (threadCount != null) {
+      tng.setThreadCount(threadCount);
+    }
+    if (parallelMode != null) {
+      tng.setParallel(parallelMode);
+    }
+    tng.setTestSuites(paths);
+    tng.addListener((ITestNGListener) tla);
+    if (null != randomizeSuites) {
+      tng.setRandomizeSuites(randomizeSuites);
+    }
+
+    BaseThreadTest.initThreadLog();
+    tng.run();
+
+    Assert.assertEquals(
+        BaseThreadTest.getThreadCount(),
+        expectedThreadCount,
+        "Thread count expected:"
+            + expectedThreadCount
+            + " actual:"
+            + BaseThreadTest.getThreadCount());
+    Assert.assertEquals(
+        BaseThreadTest.getSuitesMap().keySet().size(),
+        expectedSuiteCount,
+        "Suite count is incorrect");
+  }
+
+  private static String getPathToParallelResource(String resourceName) {
+    return getPathToResource(String.format("parallel-suites/%s", resourceName));
   }
 }
diff --git a/testng-core/src/test/resources/parallel-suites/inherit-thread-count-child.yaml b/testng-core/src/test/resources/parallel-suites/inherit-thread-count-child.yaml
@@ -0,0 +1,7 @@
+name: Child suite
+
+tests:
+  - name: Tests
+    classes:
+      - test.thread.Sample2
+      - test.thread.Sample1
diff --git a/testng-core/src/test/resources/parallel-suites/inherit-thread-count-parent.yaml b/testng-core/src/test/resources/parallel-suites/inherit-thread-count-parent.yaml
@@ -0,0 +1,5 @@
+name: Parent suite
+
+suiteFiles:
+  - ./inherit-thread-count-child.yaml
+
diff --git a/...e/src/test/resources/suite-parallel-0.xml → ...rallel-suites/simple-suite-parallel-0.xml b/...e/src/test/resources/suite-parallel-0.xml → ...rallel-suites/simple-suite-parallel-0.xml
@@ -3,13 +3,13 @@
 <suite name="Suite Parallel 0">
 
   <suite-files>
-    <suite-file path="./suite-parallel-1.xml" />
-    <suite-file path="./suite-parallel-2.xml" />
+    <suite-file path="simple-suite-parallel-1.xml" />
+    <suite-file path="simple-suite-parallel-2.xml" />
   </suite-files>
 
   <test name="Test0">
     <classes>
       <class name="test.thread.Sample2"/>
     </classes>
   </test>
-</suite>
+</suite>
diff --git a/...e/src/test/resources/suite-parallel-1.xml → ...rallel-suites/simple-suite-parallel-1.xml b/...e/src/test/resources/suite-parallel-1.xml → ...rallel-suites/simple-suite-parallel-1.xml
diff --git a/...e/src/test/resources/suite-parallel-2.xml → ...rallel-suites/simple-suite-parallel-2.xml b/...e/src/test/resources/suite-parallel-2.xml → ...rallel-suites/simple-suite-parallel-2.xml
diff --git a/testng-core/src/test/resources/testng.xml b/testng-core/src/test/resources/testng.xml
@@ -109,9 +109,7 @@
       <class name="test.github765.ExcludeSyntheticMethodsFromTemplateCallsTest"/>
       <class name="test.github1405.TestExclusionOfMainMethod"/>
       <class name="test.cli.github1517.ExitCodeListenerBehaviorTest"/>
-<!--
       <class name="test.thread.ParallelSuiteTest"/>
--->
       <class name="test.simple.IncludedExcludedTest" />
       <class name="test.reports.ReportTest" />
       <class name="test.reports.XmlReporterTest"/>

diff --git a/testng-core/src/test/resources/testng.yaml b/testng-core/src/test/resources/testng.yaml
@@ -50,6 +50,7 @@ tests:
       - test.thread.ThreadPoolSizeTest
       - test.thread.SequentialTest
       - test.thread.ParallelTestTest
+      - test.thread.ParallelSuiteTest
       - test.thread.FactoryTest
       - test.thread.DataProviderThreadPoolSizeTest
       - test.thread.MultiThreadedDependentTest