Merge branch 'topic/default/pdm' into 'branch/default'

Use PDM and build a dedicated Docker image

See merge request fluiddyn/fluidsim!349

.gitlab-ci.yml

@@ -1,15 +1,67 @@
+stages:
+  - image
+  - lint
+  - test
+
 variables:
   CODECOV_TOKEN: 4d2d8534-60ec-48b3-bf55-93b92f25913d
 
-tests:
-  image: fluiddyn/python3-stable:lastest
+image: registry.heptapod.net:443/fluiddyn/fluidsim/ci/default:stable
+
+# check_bug:
+#   image: fluiddyn/python3-stable:lastest
+#   script:
+#     - pip index versions flit-core
+#     - pip install requests
+#     - python3.9 tmp_bug_unearth.py
+
+# Build an image for the other tasks; this should be a scheduled job, as
+# it is quite unnecessary to run on every invocation.
+CI image:
+  stage: image
+  tags:
+    - container-registry-push
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+    - if: '$CI_BUILD_IMAGES == "1"'
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [ "" ]
+  before_script:
+    - ""
   script:
-    - pip install -U pip tox --user
-    - tox -e py39,py39-fft,codecov
+    - |
+      cat > /kaniko/.docker/config.json <<EOF
+      {
+        "auths": {
+          "$CI_REGISTRY": {
+            "username": "$CI_REGISTRY_USER",
+            "password": "$CI_REGISTRY_PASSWORD"
+          }
+        }
+      }
+      EOF
+    - >
+      /kaniko/executor --context $CI_PROJECT_DIR
+      --dockerfile $CI_PROJECT_DIR/docker/Dockerfile
+      --single-snapshot
+      --cleanup
+      --destination registry.heptapod.net:443/fluiddyn/fluidsim/ci/$CI_COMMIT_HG_BRANCH:stable
 
-lint:
-  image: fluiddyn/python3-stable:lastest
+validate_code:
+  stage: lint
+  needs:
+    - job: "CI image"
+      optional: true
   script:
-    - pip install -U pip tox --user
-    - tox -e lint
-    - tox -e black
+    - pdm install -G dev --no-self
+    - pdm run make lint
+    - pdm run make black_check
+
+tests:
+  stage: test
+  needs:
+    - job: "CI image"
+      optional: true
+  script:
+    - tox -e py39,py39-fft,codecov

.hgignore

@@ -85,3 +85,4 @@ fluidsim/_version.py
 doc/examples/htmlcov/*
 
 fluidsim/build_conf.txt
+.pdm-python

Makefile

@@ -5,13 +5,9 @@ MPI_NUM_PROCS ?= 2
 
 .PHONY: black black_check clean clean_pyc clean_so cleantransonic coverage_short develop develop_lib develop_user dist lint _report_coverage shortlog tests _tests_coverage tests_mpi
 
-develop: develop_lib
-	pip install meson-python ninja numpy "pythran>=0.9.7"
-	pip install --force-reinstall transonic@hg+https://foss.heptapod.net/fluiddyn/transonic
-	pip install -e ".[dev]" --no-build-isolation
-
-develop_lib:
-	cd lib && pip install -e .
+develop:
+	pdm install --no-self
+	pdm run pip install -e . --no-build-isolation
 
 dist:
 	pip install build

docker/Dockerfile

@@ -0,0 +1,62 @@
+FROM python:3.9
+LABEL Pierre Augier <[email protected]>
+
+RUN apt-get update
+
+RUN apt-get install -y --no-install-recommends \
+  libfftw3-dev libfftw3-mpi-dev \
+  openmpi-bin libhdf5-openmpi-dev libopenblas-dev
+RUN apt-get install -y --no-install-recommends \
+  gfortran clang
+# also include utilities for debugging
+RUN apt-get install -y --no-install-recommends \
+  emacs vim \
+  meld less \
+  # python3-pip is for Mercurial (hg-git, hg-evolve)
+  mercurial python3-pip git
+RUN apt-get install -y --no-install-recommends \
+  rsync \
+  libgl1 xvfb xauth libgl1-mesa-dev \
+  libxkbcommon-x11-0 \
+  swig sudo
+
+RUN rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 1000 appuser && useradd -m -r -u 1000 -g appuser -s /bin/bash appuser -s /bin/bash && usermod -a -G sudo appuser
+RUN echo appuser:appuser | chpasswd
+USER appuser
+ARG HOME=/home/appuser
+RUN mkdir -p $HOME/opt
+WORKDIR $HOME/opt
+RUN echo $USER $HOME $PWD && whoami
+
+RUN $(hg debuginstall -T '{pythonexe}') -m pip install hg-evolve hg-git --no-cache-dir --user --break-system-packages
+
+COPY --chown=appuser:appuser docker/hgrc $HOME/.hgrc
+COPY --chown=appuser:appuser docker/pythranrc $HOME/.pythranrc
+
+RUN mkdir -p $HOME/.local/include
+RUN mkdir -p $HOME/.local/lib
+RUN ln -s /usr/include/fftw* $HOME/.local/include
+RUN ln -s /usr/lib/x86_64-linux-gnu/libfftw3* $HOME/.local/lib
+
+ENV LD_LIBRARY_PATH=$HOME/.local/lib
+ENV PATH=$HOME/.local/bin:$PATH
+ENV CPATH=$HOME/.local/include:$CPATH
+
+RUN mkdir -p $HOME/.config/matplotlib
+RUN echo 'backend      : agg' > $HOME/.config/matplotlib/matplotlibrc
+
+RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/site.cfg.files/site.cfg.docker -O ~/.fluidfft-site.cfg
+
+RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_p3dfft.sh -O ./install_p3dfft.sh
+RUN chmod +x install_p3dfft.sh
+RUN export FCFLAGS="-w -fallow-argument-mismatch -O2" && \
+    export FFLAGS="-w -fallow-argument-mismatch -O2" && \
+    ./install_p3dfft.sh
+
+RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_pfft.sh -O ./install_pfft.sh
+RUN chmod +x install_pfft.sh
+RUN ./install_pfft.sh
+
+RUN pip install -U pip pdm tox tox-pdm --user

docker/Makefile

@@ -0,0 +1,49 @@
+image := image-fluidsim
+tag := $(shell date -I'date'| tr -d "[:punct:]")
+
+define STR_HELP
+This makefile can be used to build images and start containers.
+
+Examples
+--------
+$ make start  # starts Docker service
+$ make pull   # pulls a Docker image fron Docker hub
+$ make build  # by default builds python3-stable image
+$ make run
+$ make build image=python3-stable
+$ make cleanall  # clean (unnecessary, not all) containers and images
+
+endef
+
+export STR_HELP
+
+.PHONY: help
+
+help:
+	@echo "$$STR_HELP"
+
+start:
+	systemctl start docker
+
+build:
+	docker build -f Dockerfile -t fluiddyn/$(image) ..
+	docker tag fluiddyn/$(image) fluiddyn/$(image):$(tag)
+
+list:
+	@printf "\nImages: "
+	docker images
+	@printf "\nContainers: "
+	docker ps
+
+cleancontainers:
+	@echo "Clean exited containers."
+	for cont in $$(docker ps -a | awk 'NR>1{print $$1}'); do docker stop $$cont; docker rm $$cont; done
+
+cleanimages:
+	@echo "Clean dangling images with no tag."
+	for img in $$(docker images -qf "dangling=true"); do docker rmi -f $$img; done
+
+cleanall: cleancontainers cleanimages cleanmako
+
+run:
+	docker run --name $(image) --restart always -it fluiddyn/$(image) bash

docker/hgrc

@@ -0,0 +1,28 @@
+# example user config (see 'hg help config' for more info)
+[ui]
+# name and email, e.g.
+# username = Jane Doe <[email protected]>
+editor = emacs -nw -Q
+
+# We recommend enabling tweakdefaults to get slight improvements to
+# the UI over time. Make sure to set HGPLAIN in the environment when
+# writing scripts!
+tweakdefaults = True
+
+[extensions]
+# uncomment these lines to enable some popular extensions
+# (see 'hg help extensions' for more info)
+#
+churn =
+shelve =
+hgext.extdiff =
+rebase =
+absorb =
+evolve =
+topic =
+
+[extdiff]
+cmd.meld =
+
+[trusted]
+users = root

docker/pythranrc

@@ -0,0 +1,7 @@
+[pythran]
+complex_hook = True
+
+[compiler]
+CXX=clang++
+CC=clang
+blas=openblas