refactor test framework to support docker tests

org.lflang.tests/src/org/lflang/tests/AbstractTest.java

@@ -56,6 +56,13 @@ protected boolean supportsGenericTypes() {
         return false;
     }
 
+    /**
+     * Whether to enable {@link #runDockerNonfederatedTests()} and {@link #runDockerFederatedTests()}.
+     */
+    protected boolean supportsDockerOption() {
+        return false;
+    }
+
 
     @Test
     public void runExampleTests() {
@@ -145,6 +152,25 @@ public void runFederatedTests() {
                            false);
     }
 
+    @Test
+    public void runDockerNonfederatedTests() {
+        Assumptions.assumeTrue(isLinux(), Message.NO_DOCKER_TEST_SUPPORT);
+        Assumptions.assumeTrue(supportsDockerOption(), Message.NO_DOCKER_SUPPORT);
+        runTestsForTargets(Message.DESC_DOCKER_NONFEDERATED,
+                           TestCategory.DOCKER_NONFEDERATED::equals, Configurators::noChanges, TestLevel.EXECUTION,
+                           false);
+    }
+
+    @Test
+    public void runDockerFederatedTests() {
+        Assumptions.assumeTrue(isLinux(), Message.NO_DOCKER_TEST_SUPPORT);
+        Assumptions.assumeTrue(supportsDockerOption(), Message.NO_DOCKER_SUPPORT);
+        Assumptions.assumeTrue(supportsFederatedExecution(), Message.NO_FEDERATION_SUPPORT);
+        runTestsForTargets(Message.DESC_DOCKER_FEDERATED,
+                           TestCategory.DOCKER_FEDERATED::equals, Configurators::noChanges, TestLevel.EXECUTION,
+                           false);
+    }
+
 
     @Test
     public void runWithFourThreads() {

org.lflang.tests/src/org/lflang/tests/Configurators.java

@@ -82,17 +82,17 @@ static boolean noChanges(LFTest test) {
      * categories.
      */
     public static boolean defaultCategoryExclusion(TestCategory category) {
-        if (category != TestCategory.CONCURRENT && category != TestCategory.FEDERATED &&
-            category != TestCategory.EXAMPLE) {
-            // Check if running on Windows
-            if (TestBase.isWindows()) {
-                // SERIALIZATION and TARGET tests are currently not
-                // supported on Windows.
-                return category != TestCategory.SERIALIZATION &&
-                    category != TestCategory.TARGET;
-            }
-            return true;
-        }
-        return false;
+        boolean excluded = false;
+
+        // CONCURRENT, FEDERATED, EXAMPLE, DOCKER_FEDERATED, DOCKER_NONFEDERATED are excluded
+        excluded |= (category == TestCategory.CONCURRENT 
+                    || category == TestCategory.FEDERATED 
+                    || category == TestCategory.EXAMPLE 
+                    || category == TestCategory.DOCKER_FEDERATED 
+                    || category == TestCategory.DOCKER_NONFEDERATED);
+
+        // SERIALIZATION and TARGET tests are excluded on Windows.
+        excluded |= (TestBase.isWindows() && (category == TestCategory.SERIALIZATION || category == TestCategory.TARGET));
+        return !excluded;
     }
 }

org.lflang.tests/src/org/lflang/tests/TestBase.java

@@ -8,9 +8,17 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FileFilter;
+import java.io.BufferedWriter;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Objects;
 import java.util.Properties;
 import java.util.Set;
@@ -114,6 +122,8 @@ public static class Message {
         public static final String ALWAYS_MULTITHREADED = "The reactor-cpp runtime is always multithreaded.";
         public static final String NO_THREAD_SUPPORT = "Target does not support the 'threads' property.";
         public static final String NO_FEDERATION_SUPPORT = "Target does not support federated execution.";
+        public static final String NO_DOCKER_SUPPORT = "Target does not support the 'docker' property.";
+        public static final String NO_DOCKER_TEST_SUPPORT = "Docker tests are only supported on Linux.";
         public static final String NO_GENERICS_SUPPORT = "Target does not support generic types.";
 
         /* Descriptions of collections of tests. */
@@ -125,6 +135,8 @@ public static class Message {
         public static final String DESC_MULTIPORT = "Run multiport tests (threads = 0).";
         public static final String DESC_AS_FEDERATED = "Run non-federated tests in federated mode.";
         public static final String DESC_FEDERATED = "Run federated tests.";
+        public static final String DESC_DOCKER_NONFEDERATED = "Run docker non-federated tests.";
+        public static final String DESC_DOCKER_FEDERATED = "Run docker federated tests.";
         public static final String DESC_CONCURRENT = "Run concurrent tests.";
         public static final String DESC_TARGET_SPECIFIC = "Run target-specific tests (threads = 0)";
         public static final String DESC_AS_CCPP = "Running C tests as CCpp.";
@@ -232,6 +244,24 @@ protected static boolean isWindows() {
         return OS.contains("win");
     }
 
+     /**
+     * Determine whether the current platform is MacOS.
+     * @return true if the current platform is MacOS, false otherwise.
+     */
+    protected static boolean isMac() {
+        String OS = System.getProperty("os.name").toLowerCase();
+        return OS.contains("mac");
+    }
+
+    /**
+     * Determine whether the current platform is Linux.
+     * @return true if the current platform is Linux, false otherwise.
+     */
+    protected static boolean isLinux() {
+        String OS = System.getProperty("os.name").toLowerCase();
+        return OS.contains("linux");
+    }
+
     /**
      * End output redirection.
      */
@@ -426,48 +456,163 @@ private void generateCode(LFTest test) {
      * an error code.
      */
     private void execute(LFTest test) {
-        final ProcessBuilder pb = getExecCommand(test);
-        if (pb == null) {
+        final List<ProcessBuilder> pbList = getExecCommand(test);
+        if (pbList.isEmpty()) {
             return;
         }
         try {
-            var p = pb.start();
-            var stdout = test.execLog.recordStdOut(p);
-            var stderr = test.execLog.recordStdErr(p);
-            if (!p.waitFor(MAX_EXECUTION_TIME_SECONDS, TimeUnit.SECONDS)) {
-                stdout.interrupt();
-                stderr.interrupt();
-                p.destroyForcibly();
-                test.result = Result.TEST_TIMEOUT;
-            } else {
-                if (p.exitValue() == 0) {
-                    test.result = Result.TEST_PASS;
+            for (ProcessBuilder pb : pbList) {
+                var p = pb.start();
+                var stdout = test.execLog.recordStdOut(p);
+                var stderr = test.execLog.recordStdErr(p);
+                if (!p.waitFor(MAX_EXECUTION_TIME_SECONDS, TimeUnit.SECONDS)) {
+                    stdout.interrupt();
+                    stderr.interrupt();
+                    p.destroyForcibly();
+                    test.result = Result.TEST_TIMEOUT;
+                    return;
                 } else {
-                    test.result = Result.TEST_FAIL;
+                    if (p.exitValue() != 0) {
+                        test.result = Result.TEST_FAIL;
+                        return;
+                    }
                 }
             }
-
         } catch (Exception e) {
             test.result = Result.TEST_FAIL;
+            return;
+        }
+        test.result = Result.TEST_PASS;
+    }
+
+    /**
+     * Return a Mapping of federateName -> absolutePathToDockerFile.
+     * Expects the docker file to be two levels below where the source files are generated (ex. srcGenPath/nameOfFederate/*Dockerfile)
+     * @param test The test to get the execution command for.
+     */
+    private Map<String, Path> getFederatedDockerFiles(LFTest test) {
+        Map<String, Path> fedNameToDockerFile = new HashMap<>();
+        File[] srcGenFiles = test.fileConfig.getSrcGenPath().toFile().listFiles();
+        for (File srcGenFile : srcGenFiles) {
+            if (srcGenFile.isDirectory()) {
+                File[] dockerFile = srcGenFile.listFiles(new FileFilter() {
+                    @Override
+                    public boolean accept(File pathName) {
+                        return pathName.getName().endsWith("Dockerfile");
+                    }
+                });
+                assert dockerFile.length == 1;
+                fedNameToDockerFile.put(srcGenFile.getName(), dockerFile[0].getAbsoluteFile().toPath());
+            }
+        }
+        return fedNameToDockerFile;
+    }
+
+    /**
+     * Return the content of the bash script used for testing docker option in federated execution. 
+     * @param fedNameToDockerFile A mapping of federateName -> absolutePathToDockerFile
+     * @param testNetworkName The name of the network used for testing the docker option. 
+     *                        See https://github.com/lf-lang/lingua-franca/wiki/Containerized-Execution#federated-execution for more details.
+     */
+    private String getDockerRunScript(Map<String, Path> fedNameToDockerFile, String testNetworkName) {
+        StringBuilder shCode = new StringBuilder();
+        shCode.append("#!/bin/bash\n");
+        int n = fedNameToDockerFile.size();
+        shCode.append("pids=\"\"\n");
+        shCode.append(String.format("docker run --rm --network=%s --name=rti rti:test -i 1 -n %d &\n", testNetworkName, n));
+        shCode.append("pids+=\"$!\"\nsleep 3\n");
+        for (String fedName : fedNameToDockerFile.keySet()) {
+            Path dockerFile = fedNameToDockerFile.get(fedName);
+            shCode.append(String.format("docker run --rm --network=%s %s:test -i 1 &\n", testNetworkName, fedName));
+            shCode.append("pids+=\" $!\"\n");
+        }
+        shCode.append("for p in $pids; do\n");
+        shCode.append("    if wait $p; then\n");
+        shCode.append("        :\n");
+        shCode.append("    else\n");
+        shCode.append("        exit 1\n");
+        shCode.append("    fi\n");
+        shCode.append("done\n");
+        return shCode.toString();
+    }
+
+    /**
+     * Return a list of ProcessBuilders used to test the docker option under non-federated execution.
+     * @param test The test to get the execution command for.
+     */
+    private List<ProcessBuilder> getNonfederatedDockerExecCommand(LFTest test) {
+        var srcGenPath = test.fileConfig.getSrcGenPath();
+        var dockerPath = srcGenPath.resolve(test.fileConfig.name + ".Dockerfile");
+        return Arrays.asList(new ProcessBuilder("docker", "build", "-t", "lingua_franca:test", "-f", dockerPath.toString(), srcGenPath.toString()), 
+                             new ProcessBuilder("docker", "run", "--rm", "lingua_franca:test"),
+                             new ProcessBuilder("docker", "image", "rm", "lingua_franca:test"));
+    }
+
+    /**
+     * Return a list of ProcessBuilders used to test the docker option under federated execution.
+     * @param test The test to get the execution command for.
+     */
+    private List<ProcessBuilder> getFederatedDockerExecCommand(LFTest test) {
+        var rtiPath = test.fileConfig.getSrcGenBasePath().resolve("RTI");
+        var rtiDockerPath = rtiPath.resolve("rti.Dockerfile");
+        Map<String, Path> fedNameToDockerFile = getFederatedDockerFiles(test);
+        try {
+            File testScript = File.createTempFile("dockertest", null);
+            testScript.deleteOnExit();
+            if (!testScript.setExecutable(true)) {
+                throw new IOException("Failed to make test script executable");
+            }
+            FileWriter fileWriter = new FileWriter(testScript.getAbsoluteFile(), true);
+            String testNetworkName = "linguaFrancaTestNetwork";
+            BufferedWriter bufferedWriter = new BufferedWriter(fileWriter);
+            bufferedWriter.write(getDockerRunScript(fedNameToDockerFile, testNetworkName));
+            bufferedWriter.close();
+            List<ProcessBuilder> execCommands = new ArrayList<>();
+            execCommands.add(new ProcessBuilder("docker", "network", "create", testNetworkName));
+            execCommands.add(new ProcessBuilder("docker", "build", "-t", "rti:test", "-f", rtiDockerPath.toString(), rtiPath.toString()));
+            for (String fedName : fedNameToDockerFile.keySet()) {
+                Path dockerFile = fedNameToDockerFile.get(fedName);
+                execCommands.add(new ProcessBuilder("docker", "build", "-t", fedName + ":test", "-f", dockerFile.toString(), dockerFile.getParent().toString()));
+            }
+            execCommands.add(new ProcessBuilder(testScript.getAbsolutePath()));
+            execCommands.add(new ProcessBuilder("docker", "image", "rm", "rti:test"));
+            for (String fedName : fedNameToDockerFile.keySet()) {
+                Path dockerFile = fedNameToDockerFile.get(fedName);
+                execCommands.add(new ProcessBuilder("docker", "image", "rm", fedName + ":test"));
+            }
+            execCommands.add(new ProcessBuilder("docker", "network", "rm", testNetworkName));
+            return execCommands;
+        } catch (IOException e) {
+            return Arrays.asList(new ProcessBuilder("exit", "1"));
         }
     }
 
     /**
-     * Return a preconfigured ProcessBuilder for the command
+     * Return a list of preconfigured ProcessBuilder(s) for the command(s)
      * that should be used to execute the test program.
      * @param test The test to get the execution command for.
      */
-    private ProcessBuilder getExecCommand(LFTest test) {
+    private List<ProcessBuilder> getExecCommand(LFTest test) {
         final var nameWithExtension = test.srcFile.getFileName().toString();
         final var nameOnly = nameWithExtension.substring(0, nameWithExtension.lastIndexOf('.'));
+
+        var srcGenPath = test.fileConfig.getSrcGenPath();
+        var parentDirName = srcGenPath.getParent().getFileName().toString();
+        // special case to test docker file generation
+        if (parentDirName.equalsIgnoreCase(TestCategory.DOCKER_NONFEDERATED.name())) {
+            return getNonfederatedDockerExecCommand(test);
+        } else if (parentDirName.equalsIgnoreCase(TestCategory.DOCKER_FEDERATED.name())) {
+            return getFederatedDockerExecCommand(test);
+        }
+
+        var binPath = test.fileConfig.binPath;
+        var binaryName = nameOnly;
 
         switch (test.target) {
         case C:
         case CPP:
         case Rust:
         case CCPP: {
-            var binPath = test.fileConfig.binPath;
-            var binaryName = nameOnly;
             if (test.target == Target.Rust) {
                 // rust binaries uses snake_case
                 binaryName = StringUtil.camelToSnakeCase(binaryName);
@@ -482,53 +627,48 @@ private ProcessBuilder getExecCommand(LFTest test) {
                 // Running the command as .\binary.exe does not work on Windows for
                 // some reason... Thus we simply pass the full path here, which
                 // should work across all platforms
-                return new ProcessBuilder(fullPath.toString()).directory(binPath.toFile());
+                return Arrays.asList(new ProcessBuilder(fullPath.toString()).directory(binPath.toFile()));
             } else {
                 test.issues.append(fullPath).append(": No such file or directory.").append(System.lineSeparator());
                 test.result = Result.NO_EXEC_FAIL;
-                return null;
+                return new ArrayList<>();
             }
         }
         case Python: {
-            var binPath = test.fileConfig.binPath;
-            var binaryName = nameOnly;
             var fullPath = binPath.resolve(binaryName);
             if (Files.exists(fullPath)) {
                 // If execution script exists, run it.
-                return new ProcessBuilder(fullPath.toString()).directory(binPath.toFile());
+                return Arrays.asList(new ProcessBuilder(fullPath.toString()).directory(binPath.toFile()));
             }
-            var srcGen = test.fileConfig.getSrcGenPath();
-            fullPath = srcGen.resolve(nameOnly + ".py");
+            fullPath = srcGenPath.resolve(nameOnly + ".py");
             if (Files.exists(fullPath)) {
-                return new ProcessBuilder("python3", fullPath.getFileName().toString())
-                    .directory(srcGen.toFile());
+                return Arrays.asList(new ProcessBuilder("python3", fullPath.getFileName().toString())
+                    .directory(srcGenPath.toFile()));
             } else {
                 test.result = Result.NO_EXEC_FAIL;
                 test.issues.append("File: ").append(fullPath).append(System.lineSeparator());
-                return null;
+                return new ArrayList<>();
             }
         }
         case TS: {
-            var binPath = test.fileConfig.binPath;
-            var binaryName = nameOnly;
             // Adjust binary extension if running on Window
             if (System.getProperty("os.name").startsWith("Windows")) {
                 binaryName += ".exe";
             }
             var fullPath = binPath.resolve(binaryName);
             if (Files.exists(fullPath)) {
                 // If execution script exists, run it.
-                return new ProcessBuilder(fullPath.toString()).directory(binPath.toFile());
+                return Arrays.asList(new ProcessBuilder(fullPath.toString()).directory(binPath.toFile()));
             }
             // If execution script does not exist, run .js directly.
             var dist = test.fileConfig.getSrcGenPath().resolve("dist");
             var file = dist.resolve(nameOnly + ".js");
             if (Files.exists(file)) {
-                return new ProcessBuilder("node", file.toString());
+                return Arrays.asList(new ProcessBuilder("node", file.toString()));
             } else {
                 test.result = Result.NO_EXEC_FAIL;
                 test.issues.append("File: ").append(file).append(System.lineSeparator());
-                return null;
+                return new ArrayList<>();
             }
         }
         default:

org.lflang.tests/src/org/lflang/tests/TestRegistry.java

@@ -152,7 +152,8 @@ public enum TestCategory {
         FEDERATED(true),
 
         // non-shared tests
-
+        DOCKER_NONFEDERATED(false),
+        DOCKER_FEDERATED(false),
         SERIALIZATION(false),
         TARGET(false),
         EXAMPLE(false),