Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

generate docker compose files for federated execution #754

Merged
merged 8 commits into from
Dec 8, 2021
Merged

generate docker compose files for federated execution #754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a6a38cd
add docker compose file generation for RTI
housengw Nov 16, 2021
38ff61e
add docker compose file for federates
housengw Nov 16, 2021
7141737
Merge branch 'docker-test' into generate-docker-compose
housengw Nov 30, 2021
1353b20
remove external netwokr
housengw Nov 30, 2021
1096c03
Merge branch 'docker-test' into generate-docker-compose
housengw Dec 3, 2021
0b9a1d1
fix dockerfile naming
housengw Dec 3, 2021
da360d1
fix docker file generation for python
housengw Dec 3, 2021
89e2f0a
merge master into generate-docker-compose
housengw Dec 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 103 additions & 20 deletions org.lflang/src/org/lflang/generator/c/CGenerator.xtend
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package org.lflang.generator.c

import java.io.File
import java.nio.file.Path
import java.util.ArrayList
import java.util.HashSet
import java.util.LinkedHashMap
Expand Down Expand Up @@ -540,6 +539,14 @@ class CGenerator extends GeneratorBase {

addPlatformFiles(coreFiles);

// TODO: Find a better way to automatically generate a unique federationID.
var dockerComposeFederationID = 1;

// TODO: Find a better way to come up with a unique network name.
var dockerComposeNetworkName = 'lf';

var dockerComposeServices = new StringBuilder();

// If there are federates, copy the required files for that.
// Also, create the RTI C file and the launcher script.
if (isFederated) {
Expand All @@ -555,12 +562,13 @@ class CGenerator extends GeneratorBase {
createFederatedLauncher(coreFiles);

if (targetConfig.dockerOptions !== null) {
var rtiPath = fileConfig.getSrcGenBasePath().resolve("RTI")
var rtiDir = rtiPath.toFile()
var rtiDir = fileConfig.getSrcGenBasePath().resolve("RTI").toFile()
if (!rtiDir.exists()) {
rtiDir.mkdirs()
}
writeRTIDockerFile(rtiPath, rtiDir)
var dockerFileName = 'rti.Dockerfile'
writeRTIDockerFile(rtiDir, dockerFileName)
writeRTIDockerComposeFile(rtiDir, dockerFileName, dockerComposeNetworkName, dockerComposeFederationID, federates.size)
copyRtiFiles(rtiDir, coreFiles)
}
}
Expand Down Expand Up @@ -858,7 +866,9 @@ class CGenerator extends GeneratorBase {

// Create docker file.
if (targetConfig.dockerOptions !== null) {
writeDockerFile(topLevelName)
var dockerFileName = topLevelName + '.Dockerfile'
writeDockerFile(dockerFileName)
appendFederateToDockerComposeServices(dockerComposeServices, federate.name, dockerFileName, dockerComposeFederationID)
}

// If this code generator is directly compiling the code, compile it now so that we
Expand Down Expand Up @@ -894,6 +904,8 @@ class CGenerator extends GeneratorBase {
}
fileConfig = oldFileConfig;
}

writeFederatesDockerComposeFile(fileConfig.getSrcGenPath().toFile(), dockerComposeServices, dockerComposeNetworkName);

// Initiate an orderly shutdown in which previously submitted tasks are
// executed, but no new tasks will be accepted.
Expand Down Expand Up @@ -1256,7 +1268,7 @@ class CGenerator extends GeneratorBase {
*/
override writeDockerFile(String dockerFileName) {
var srcGenPath = fileConfig.getSrcGenPath
val dockerFile = srcGenPath + File.separator + dockerFileName + '.Dockerfile'
val dockerFile = srcGenPath + File.separator + dockerFileName
// If a dockerfile exists, remove it.
var file = new File(dockerFile)
if (file.exists) {
Expand All @@ -1280,13 +1292,9 @@ class CGenerator extends GeneratorBase {
if (!targetConfig.fileNames.nullOrEmpty) {
additionalFiles = '''COPY "«targetConfig.fileNames.join('" "')»" "src-gen/"'''
}
var dockerCompiler = 'gcc'
var fileExtension = 'c'

if (CCppMode) {
dockerCompiler = 'g++'
fileExtension = 'cpp'
}
var dockerCompiler = CCppMode ? 'g++' : 'gcc'
var fileExtension = CCppMode ? 'cpp' : 'c'
var setRtiHostName = isFederated ? '''ENV RTI_HOST=«federationRTIProperties.get('host').toString»''' : ''

pr(contents, '''
# Generated docker file for «topLevelName» in «srcGenPath».
Expand All @@ -1296,6 +1304,7 @@ class CGenerator extends GeneratorBase {
RUN set -ex && apk add --no-cache «dockerCompiler» musl-dev cmake make
COPY core src-gen/core
COPY ctarget.h ctarget.c src-gen/
«setRtiHostName»
COPY CMakeLists.txt \
«topLevelName».«fileExtension» src-gen/
«additionalFiles»
Expand Down Expand Up @@ -1323,13 +1332,50 @@ class CGenerator extends GeneratorBase {
''')
}

/**
* Write the docker-compose.yml for orchestrating the federates.
* @param the directory to write the docker-compose.yml
* @param content of the "services" section of the docker-compose.yml
* @param the name of the network hosting the federation
*/
def writeFederatesDockerComposeFile(File dir, StringBuilder dockerComposeServices, String networkName) {
val dockerComposeFileName = 'docker-compose.yml'
val dockerComposeFile = dir + File.separator + dockerComposeFileName
val contents = new StringBuilder()
pr(contents, '''
version: "3.9"
services:
«dockerComposeServices.toString»
networks:
default:
name: «networkName»
''')
JavaGeneratorUtils.writeSourceCodeToFile(contents, dockerComposeFile)
}

/**
* Append a service to the "services" section of the docker-compose.yml file.
* @param the content of the "services" section of the docker-compose.yml file.
* @param the name of the federate to be added to "services".
* @param the name of the federate's Dockerfile.
* @param the federationID.
*/
def appendFederateToDockerComposeServices(StringBuilder dockerComposeServices, String federateName, String dockerFileName, int dockerComposeFederationID) {
val tab = ' '
dockerComposeServices.append('''«tab»«federateName»:«System.lineSeparator»''')
dockerComposeServices.append('''«tab»«tab»build:«System.lineSeparator»''')
dockerComposeServices.append('''«tab»«tab»«tab»context: «federateName»«System.lineSeparator»''')
dockerComposeServices.append('''«tab»«tab»«tab»dockerfile: «dockerFileName»«System.lineSeparator»''')
dockerComposeServices.append('''«tab»«tab»command: -i «dockerComposeFederationID»«System.lineSeparator»''')
}

/**
* Write a Dockerfile for the RTI at rtiDir.
* The file will go into src-gen/RTI/rti.Dockerfile.
* @param the directory where rti.Dockerfile will be written to.
* @param name of the Dockerfile for the RTI.
*/
def writeRTIDockerFile(Path rtiPath, File rtiDir) {
val dockerFileName = 'rti.Dockerfile'
def writeRTIDockerFile(File rtiDir, String dockerFileName) {
val dockerFile = rtiDir + File.separator + dockerFileName
// If a dockerfile exists, remove it.
var file = new File(dockerFile)
Expand Down Expand Up @@ -1358,14 +1404,51 @@ class CGenerator extends GeneratorBase {
ENTRYPOINT ["./build/RTI"]
''')
JavaGeneratorUtils.writeSourceCodeToFile(contents, dockerFile)
println("Dockerfile for RTI written to " + dockerFile)
}

/**
* Write a docker-compose.yml for the RTI at rtiDir.
* The file will go into src-gen/RTI/docker-compose.yml.
* @param the directory where docker-compose.yml will be written to.
* @param name of the Dockerfile created for the RTI.
* @param name of the docker network to host the federation
* @param the federationID, which is the number passed by the -i flag to the RTI.
* @param the total number of federates.
*/
def writeRTIDockerComposeFile(File rtiDir, String rtiDockerFileName, String networkName, int federationID, int n) {
val dockerComposeFileName = 'docker-compose.yml'
val dockerComposeFile = rtiDir + File.separator + dockerComposeFileName
// If a dockerfile exists, remove it.
var file = new File(dockerComposeFile)
if (file.exists) {
file.delete
}
if (this.mainDef === null) {
return
}
val contents = new StringBuilder()
pr(contents, '''
# Generated docker-comopose file for RTI in «rtiDir».
# For instructions, see: https://github.com/icyphy/lingua-franca/wiki/Containerized-Execution
version: "3.9"
services:
«federationRTIProperties.get('host').toString»:
build:
context: .
dockerfile: «rtiDockerFileName»
command: -i «federationID» -n «n»
networks:
default:
name: «networkName»
''')
JavaGeneratorUtils.writeSourceCodeToFile(contents, dockerComposeFile)
println('''
#####################################
To build the docker image, use:
#############################################
To build the docker image of the rti, use:

docker build -t rti -f «dockerFile» «rtiDir»
docker compose -f «dockerComposeFile» up

#####################################
#############################################
''')
}

Expand Down
10 changes: 5 additions & 5 deletions org.lflang/src/org/lflang/generator/python/PythonGenerator.xtend
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1892,11 +1892,11 @@ class PythonGenerator extends CGenerator {
* The file will go into src-gen/filename.Dockerfile.
* If there is no main reactor, then no Dockerfile will be generated
* (it wouldn't be very useful).
* @param The root filename (without any extension).
* @param The name of the docker file.
*/
override writeDockerFile(String filename) {
override writeDockerFile(String dockerFileName) {
var srcGenPath = fileConfig.getSrcGenPath
val dockerFile = srcGenPath + File.separator + filename + '.Dockerfile'
val dockerFile = srcGenPath + File.separator + dockerFileName
// If a dockerfile exists, remove it.
var file = new File(dockerFile)
if (file.exists) {
Expand All @@ -1916,10 +1916,10 @@ class PythonGenerator extends CGenerator {
RUN set -ex && apt-get update && apt-get install -y python3-pip
COPY . src-gen
RUN cd src-gen && python3 setup.py install && cd ..
ENTRYPOINT ["python3", "src-gen/«filename».py"]
ENTRYPOINT ["python3", "src-gen/«topLevelName».py"]
''')
JavaGeneratorUtils.writeSourceCodeToFile(contents, dockerFile)
println("Dockerfile written to " + dockerFile)
println("Dockerfile for «topLevelName» written to " + dockerFile)
println('''
#####################################
To build the docker image, use:
Expand Down