Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[1.4.x] Adds additional build envs #14920

Merged
merged 3 commits into from
May 9, 2019
Merged

[1.4.x] Adds additional build envs #14920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
08be9e9
Adds CUDA build environment for version 8.0, 9.0, 9.2 and 10.0
perdasilva May 8, 2019
311063f
Renames ubuntu_gpu environment to ubuntu_gpu_cu91 and updates references
perdasilva May 8, 2019
d10ddbd
Adds static build image and updates ci/build.py
perdasilva May 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ core_logic: {
ws('workspace/build-cmake-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false)
utils.docker_run('ubuntu_gpu_cu91', 'build_ubuntu_gpu_cmake_mkldnn', false)
utils.pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
}
}
Expand All @@ -371,7 +371,7 @@ core_logic: {
ws('workspace/build-cmake-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
utils.docker_run('ubuntu_gpu_cu91', 'build_ubuntu_gpu_cmake', false)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do CU91 here? Scala do CU92 as a default

Copy link
Contributor Author

@perdasilva perdasilva May 9, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately, until very recently (#14513), CU91 was the version CI used (with the exception of the Scala stuff you guys added). I'm sticking with CU91 because that's what ubuntu_gpu is in this branch (see here).

CI in this branch definitely deserves an update. It should be brought to CUDA 10 (and once stu1130 is finished getting cudnn 7.5. working, CUDA 10.1). But I think this requires more time than I have atm.

utils.pack_lib('cmake_gpu', mx_cmake_lib, true)
}
}
Expand Down Expand Up @@ -542,7 +542,7 @@ core_logic: {
ws('workspace/ut-python2-gpu') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
python2_gpu_ut('ubuntu_gpu')
python2_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml')
Expand All @@ -555,7 +555,7 @@ core_logic: {
ws('workspace/ut-python3-gpu') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
python3_gpu_ut('ubuntu_gpu')
python3_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml')
Expand All @@ -569,7 +569,7 @@ core_logic: {
timeout(time: max_time, unit: 'MINUTES') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true)
utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_python2_quantization_gpu', true)
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml')
Expand All @@ -584,7 +584,7 @@ core_logic: {
timeout(time: max_time, unit: 'MINUTES') {
try {
utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true)
utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_python3_quantization_gpu', true)
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python3_quantize_gpu.xml')
Expand Down Expand Up @@ -613,7 +613,7 @@ core_logic: {
ws('workspace/ut-python2-mkldnn-gpu') {
try {
utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
python2_gpu_ut('ubuntu_gpu')
python2_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml')
Expand All @@ -640,7 +640,7 @@ core_logic: {
ws('workspace/ut-python3-mkldnn-gpu') {
try {
utils.unpack_and_init('mkldnn_gpu', mx_mkldnn_lib, true)
python3_gpu_ut('ubuntu_gpu')
python3_gpu_ut('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu.xml')
Expand All @@ -653,7 +653,7 @@ core_logic: {
ws('workspace/ut-python3-mkldnn-gpu-nocudnn') {
try {
utils.unpack_and_init('mkldnn_gpu_nocudnn', mx_mkldnn_lib, true)
python3_gpu_ut_nocudnn('ubuntu_gpu')
python3_gpu_ut_nocudnn('ubuntu_gpu_cu91')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml')
Expand Down Expand Up @@ -756,7 +756,7 @@ core_logic: {
ws('workspace/ut-perl-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_cpugpu_perl', true)
utils.publish_test_coverage()
}
}
Expand All @@ -767,7 +767,7 @@ core_logic: {
ws('workspace/ut-cpp-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cmake_gpu', mx_cmake_lib, true)
utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
Expand All @@ -778,7 +778,7 @@ core_logic: {
ws('workspace/ut-cpp-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib, true)
utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_cpp', true)
utils.publish_test_coverage()
}
}
Expand All @@ -800,7 +800,7 @@ core_logic: {
ws('workspace/ut-r-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true)
utils.docker_run('ubuntu_gpu_cu91', 'unittest_ubuntu_gpu_R', true)
utils.publish_test_coverage()
}
}
Expand Down Expand Up @@ -911,7 +911,7 @@ core_logic: {
ws('workspace/it-python-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true)
utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_python', true)
utils.publish_test_coverage()
}
}
Expand All @@ -922,7 +922,7 @@ core_logic: {
ws('workspace/it-cpp-package') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib_cpp_examples, true)
utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true)
utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_cpp_package', true)
utils.publish_test_coverage()
}
}
Expand All @@ -935,7 +935,7 @@ core_logic: {
// timeout(time: max_time, unit: 'MINUTES') {
// utils.init_git()
// utils.unpack_lib('gpu', mx_lib)
// utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true)
// utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_caffe', true)
// utils.publish_test_coverage()
// }
// }
Expand All @@ -946,7 +946,7 @@ core_logic: {
ws('workspace/it-dist-kvstore') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_lib, true)
utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
utils.publish_test_coverage()
}
}
Expand All @@ -972,7 +972,7 @@ core_logic: {
ws('workspace/ut-scala-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init('gpu', mx_dist_lib, true)
utils.docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
utils.docker_run('ubuntu_gpu_cu91', 'integrationtest_ubuntu_gpu_scala', true)
utils.publish_test_coverage()
}
}
Expand Down
33 changes: 22 additions & 11 deletions ci/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,22 +92,24 @@ def get_dockerfiles_path():

def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
"""Get a list of architectures given our dockerfiles"""
dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles))
files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
platforms = list(map(lambda x: os.path.split(x)[1], sorted(files)))
return platforms


def get_docker_tag(platform: str, registry: str) -> str:
""":return: docker tag to be used for the container"""
platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
if not registry:
registry = "mxnet_local"
return "{0}/build.{1}".format(registry, platform)
return "{0}/{1}".format(registry, platform)


def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
return os.path.join(path, "Dockerfile.build.{0}".format(platform))
platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
return os.path.join(path, "Dockerfile.{0}".format(platform))


def get_docker_binary(use_nvidia_docker: bool) -> str:
Expand Down Expand Up @@ -200,7 +202,7 @@ def default_ccache_dir() -> str:
ccache_dir = "/tmp/_mxnet_ccache"
os.makedirs(ccache_dir, exist_ok=True)
return ccache_dir
return os.path.join(tempfile.gettempdir(), "ci_ccache")
return os.path.join(os.path.expanduser("~"), ".ccache")


def trim_container_id(cid):
Expand All @@ -215,20 +217,21 @@ def container_run(platform: str,
local_ccache_dir: str,
command: List[str],
cleanup: Cleanup,
environment: Dict[str, str],
dry_run: bool = False) -> int:
"""Run command in a container"""
container_wait_s = 600
#
# Environment setup
#
environment = {
environment.update({
'CCACHE_MAXSIZE': '500G',
'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared
'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is
# mounted
'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache
# verification.
}
})
# These variables are passed to the container to the process tree killer can find runaway
# process inside the container
# https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
Expand Down Expand Up @@ -446,6 +449,10 @@ def main() -> int:
parser.add_argument("--no-cache", action="store_true",
help="passes --no-cache to docker build")

parser.add_argument("-e", "--environment", nargs="*", default=[],
help="Environment variables for the docker container. "
"Specify with a list containing either names or name=value")

parser.add_argument("command",
help="command to run in the container",
nargs='*', action='append', type=str)
Expand Down Expand Up @@ -474,6 +481,9 @@ def signal_handler(signum, _):
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)

environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e]))
for e in args.environment])

if args.list:
print(list_platforms())
elif args.platform:
Expand All @@ -493,28 +503,29 @@ def signal_handler(signum, _):
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
elif args.print_docker_run:
command = []
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, dry_run=True, cleanup=cleanup, environment=environment)
else:
# With no commands, execute a build function for the target platform
command = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
logging.info("No command specified, trying default build: %s", ' '.join(command))
ret = container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)

if ret != 0:
logging.critical("Execution of %s failed with status: %d", command, ret)
return ret

elif args.all:
platforms = get_platforms()
platforms = [platform for platform in platforms if 'build.' in platform]
logging.info("Building for all architectures: %s", platforms)
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
Expand All @@ -535,7 +546,7 @@ def signal_handler(signum, _):
container_run(
platform=platform, nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir, cleanup=cleanup)
local_ccache_dir=args.ccache_dir, cleanup=cleanup, environment=environment)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)

Expand Down
83 changes: 83 additions & 0 deletions ci/docker/Dockerfile.build.ubuntu_gpu_cu100
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Dockerfile to run MXNet on Ubuntu 16.04 for GPU

FROM nvidia/cuda:10.0-devel-ubuntu16.04

ENV CUDNN_VERSION=7.3.1.20

WORKDIR /work/deps

COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh

COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh

COPY install/ubuntu_python.sh /work/
RUN /work/ubuntu_python.sh

COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
RUN /work/ubuntu_scala.sh

COPY install/ubuntu_r.sh /work/
COPY install/r.gpg /work/
RUN /work/ubuntu_r.sh

COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh

COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh

COPY install/ubuntu_mklml.sh /work/
RUN /work/ubuntu_mklml.sh

COPY install/ubuntu_tvm.sh /work/
RUN /work/ubuntu_tvm.sh

COPY install/ubuntu_llvm.sh /work/
RUN /work/ubuntu_llvm.sh

COPY install/ubuntu_caffe.sh /work/
RUN /work/ubuntu_caffe.sh

COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh

COPY install/ubuntu_docs.sh /work/
COPY install/docs_requirements /work/
RUN /work/ubuntu_docs.sh

COPY install/ubuntu_tutorials.sh /work/
RUN /work/ubuntu_tutorials.sh

ARG USER_ID=0
ARG GROUP_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh

COPY install/ubuntu_cudnn.sh /work/
RUN /work/ubuntu_cudnn.sh

COPY runtime_functions.sh /work/

WORKDIR /work/mxnet
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
Loading