Skip to content

[MTNN] merge tree neural network refactor #1336

[MTNN] merge tree neural network refactor

[MTNN] merge tree neural network refactor #1336

Workflow file for this run

name: test_build
on:
push:
# triggered on commit pushes on the "dev" branch
branches:
- 'dev'
# triggered on tag pushes with tags beginning with either "v" or "dev"
tags:
- 'v*'
- 'dev*'
- 'ci*'
- 'build*'
- 'test*'
pull_request:
# also triggered by pull requests on the "dev" branch
branches:
- 'dev'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
PV_TAG: v5.13.0-headless
PV_REPO: topology-tool-kit/ttk-paraview
jobs:
# ------------------#
# Test Ubuntu build #
# ------------------#
test-build-ubuntu:
runs-on: ${{ matrix.os }}
# trigger job when push event on a branch (not on a tag) only for
# the "topology-tool-kit/ttk" repository
# (pull-requests and tag pushes not affected)
if: ${{ github.repository_owner == 'topology-tool-kit' || !contains(github.ref, 'heads') }}
strategy:
matrix:
os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04]
steps:
- uses: actions/checkout@v4
name: Checkout TTK source code
- name: Install Ubuntu dependencies
run: |
sudo apt update
# TTK dependencies
sudo apt install -y \
cargo \
libboost-system-dev \
libeigen3-dev \
libgraphviz-dev \
libosmesa-dev \
libopenmpi-dev \
libsqlite3-dev \
libwebsocketpp-dev \
graphviz \
python3-sklearn \
ninja-build \
zlib1g-dev \
libqhull-dev \
dpkg-dev
cargo install sccache --version 0.4.2 --locked
echo "PATH=/root/.cargo/bin:$PATH" >> $GITHUB_ENV
- name: Install and setup sccache
uses: f3d-app/sccache-setup-action@v1
with:
key: ${{ matrix.os }}-0
- name: Install optional dependencies
uses: ./.github/actions/install-deps-unix
# TODO: try to replicate for mac
- name: Install Torch
shell: bash
run: |
wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.3.1%2Bcpu.zip
unzip libtorch-cxx11-abi-shared-with-deps-2.3.1+cpu.zip
sudo cp -R libtorch/* /usr/local/
- name: Fetch TTK-ParaView headless Debian package
run: |
wget -O ttk-paraview-headless.deb \
https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless-${{ matrix.os }}.deb
- name: Install ParaView .deb
run: |
sudo apt install ./ttk-paraview-headless.deb
- name: Create & configure TTK build directory
run: |
mkdir build
cd build
cmake \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DCMAKE_INSTALL_PREFIX=/usr \
-DTTK_BUILD_PARAVIEW_PLUGINS=TRUE \
-DTTK_BUILD_VTK_WRAPPERS=TRUE \
-DTTK_BUILD_STANDALONE_APPS=TRUE \
-DTTK_ENABLE_KAMIKAZE=TRUE \
-DTTK_ENABLE_MPI=TRUE \
-DTTK_ENABLE_CPU_OPTIMIZATION=FALSE \
-DTTK_ENABLE_SHARED_BASE_LIBRARIES=TRUE \
-DTTK_IMPLICIT_PRECONDITIONS_THRESHOLD=64*64*64 \
-GNinja \
$GITHUB_WORKSPACE
env:
CXXFLAGS: -Werror
- uses: ammaraskar/gcc-problem-matcher@master
- name: Build, install TTK & package it
run: |
cd build
cmake --build . --parallel
cpack -G DEB
sudo cmake --build . --target install
- name: Show sccache results
shell: bash
run: sccache --show-stats
- name: Run TTK tests
uses: ./.github/actions/test-ttk-unix
- name: Upload TTK .deb package for fast install in tests
uses: actions/upload-artifact@v4
with:
name: ttk-for-tests-${{ matrix.os }}.deb
path: build/ttk.deb
# ------------------#
# Tests on ttk-data #
# ------------------#
run-python-screenshots-tests:
runs-on: ${{ matrix.os }}
needs: test-build-ubuntu
strategy:
matrix:
os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04]
testSet: [pyTests, screenshotTests]
steps:
- name: Install Ubuntu dependencies
run: |
sudo apt update
- name: Install Torch
shell: bash
run: |
wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.3.1%2Bcpu.zip
unzip libtorch-cxx11-abi-shared-with-deps-2.3.1+cpu.zip
sudo cp -R libtorch/* /usr/local/
- name: Fetch TTK-ParaView headless Debian package
run: |
wget -O ttk-paraview-headless.deb \
https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless-${{ matrix.os }}.deb
- name: Install ParaView .deb
run: |
sudo apt install ./ttk-paraview-headless.deb
- name: Fetch TTK packages we just built in the test-build-ubuntu job
uses: actions/[email protected]
with:
name: ttk-for-tests-${{ matrix.os }}.deb
- name: Installing the previously built ttk package
run: |
sudo apt install ./ttk.deb
- uses: actions/checkout@v4
with:
repository: "topology-tool-kit/ttk-data"
ref: "dev"
path: "ttk-data"
name: Checkout ttk-data
- name: Resample large datasets
shell: python3 {0}
working-directory: ./ttk-data
run: |
from paraview import simple
for ds in ["ctBones.vti"]:
vti = simple.XMLImageDataReader(FileName=ds)
rsi = simple.ResampleToImage(Input=vti)
rsi.SamplingDimensions = [128, 128, 128]
simple.SaveData(ds, rsi)
- name: Run ttk-data states [NOT ENFORCED]
if: matrix.testSet == 'screenshotTests'
id: validate
continue-on-error: true
run: |
VERS=$(hostnamectl | grep "Operating System")
# Ubuntu 22.04 OOM killer also kills the parent process...
if [[ "$VERS" == *"22.04"* ]] || [[ "$VERS" == *"24.04"* ]]; then
rm ttk-data/states/mergeTreeClustering.pvsm
rm ttk-data/states/mergeTreePGA.pvsm
rm ttk-data/states/mergeTreeTemporalReduction.pvsm
rm ttk-data/states/persistentGenerators_darkSky.pvsm
fi
# remove buggy example
# related issue: https://github.com/topology-tool-kit/ttk/issues/1055
rm ttk-data/states/nestedTrackingFromOverlap.py
# remove examples which fill up the memory
rm ttk-data/states/topologicalOptimization_darkSky.pvsm
cd ttk-data/tests
mkdir output_screenshots
if ! python3 -u validate.py; then
if [[ "$VERS" == *"22.04"* ]] || [[ "$VERS" == *"24.04"* ]]; then
# weird opacity difference between the two Ubuntus
rm -f output_screenshots/clusteringKelvinHelmholtzInstabilities_1.png
rm -f output_screenshots/clusteringKelvinHelmholtzInstabilities_2.png
fi
if [ "$(ls -A output_screenshots)" ]; then
tar zcf screenshots.tar.gz output_screenshots
false
fi
fi
- name: Upload result screenshots
if: steps.validate.outcome == 'failure' && matrix.testSet == 'screenshotTests'
uses: actions/upload-artifact@v4
with:
name: screenshots-${{ matrix.os }}.tar.gz
path: ttk-data/tests/screenshots.tar.gz
retention-days: 10
- name: Run ttk-data Python scripts
if: matrix.testSet == 'pyTests'
run: |
cd ttk-data
# remove buggy example
# related issue: https://github.com/topology-tool-kit/ttk/issues/1055
rm python/nestedTrackingFromOverlap.py
# tmp removal of mpi example
rm python/mpiExample.py
# remove examples which fill up the memory
rm python/topologicalOptimization_darkSky.py
python3 -u python/run.py
- name: Test ttk-data Python scripts results [NOT ENFORCED]
if: matrix.testSet == 'pyTests'
continue-on-error: true
run: |
cd ttk-data
cat python/res.json
diff python/hashes/${{ matrix.os }}.json python/res.json
- name: Delete TTK package artifact
uses: geekyeggo/delete-artifact@v2
# delete package only once
if: matrix.testSet == 'screenshotTests'
with:
name: ttk-for-tests-${{ matrix.os }}.deb
# -----------------#
# Test macOS build #
# -----------------#
test-build-macos:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-12, macos-14]
if: ${{ github.repository_owner == 'topology-tool-kit' || !contains(github.ref, 'heads') }}
env:
DYLD_LIBRARY_PATH: /usr/local/lib
steps:
- uses: actions/checkout@v4
name: Checkout TTK source code
- name: Remove hosted Python
run: |
sudo rm -rf /Library/Frameworks/Python.framework/
sudo rm -rf /usr/local/Frameworks/Python.framework/
- name: Install macOS dependencies
run: |
# ParaView dependencies
brew reinstall python
brew install --cask xquartz
brew install llvm ninja open-mpi
# TTK dependencies
brew install boost eigen graphviz spectra sqlite zlib numpy qhull
- name: Install and setup sccache
uses: f3d-app/sccache-setup-action@v1
with:
key: macos-0
- name: Install optional dependencies
uses: ./.github/actions/install-deps-unix
- name: Fetch TTK-ParaView headless macOS binary archive
run: |
if [[ "${{ matrix.os }}" == "macos-12" ]]; then
wget -O ttk-paraview-headless.tar.gz \
https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless-${{ matrix.os }}.tar.gz
fi
if [[ "${{ matrix.os }}" == "macos-14" ]]; then
wget -O ttk-paraview-headless.tar.gz \
https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless-${{ matrix.os }}-arm64.tar.gz
fi
- name: Install ParaView
run: |
tar xzf ttk-paraview-headless.tar.gz
sudo cp -r ttk-paraview/* /usr/local
pvpython -m pip install --break-system-packages scikit-learn
- name: Set compilers as environment variables
run: |
echo "CC=$(brew --prefix llvm)/bin/clang" >> $GITHUB_ENV
echo "CXX=$(brew --prefix llvm)/bin/clang++" >> $GITHUB_ENV
- name: Create & configure TTK build directory
run: |
mkdir build
cd build
cmake \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=sccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-DTTK_BUILD_PARAVIEW_PLUGINS=TRUE \
-DTTK_BUILD_VTK_WRAPPERS=TRUE \
-DTTK_BUILD_STANDALONE_APPS=TRUE \
-DTTK_ENABLE_KAMIKAZE=TRUE \
-DTTK_ENABLE_MPI=TRUE \
-DTTK_ENABLE_CPU_OPTIMIZATION=FALSE \
-DTTK_ENABLE_SHARED_BASE_LIBRARIES=TRUE \
-DTTK_IMPLICIT_PRECONDITIONS_THRESHOLD=64*64*64 \
-GNinja \
$GITHUB_WORKSPACE
env:
CXXFLAGS: -Werror
- uses: ammaraskar/gcc-problem-matcher@master
- name: Build & install TTK
run: |
cd build
cmake --build . --parallel
sudo cmake --build . --target install
- name: Show sccache results
shell: bash
run: sccache --show-stats
- name: Set PYTHONPATH for macOS pvpython
run: |
# pvpython does not embed the correct PYTHONPATH
echo "PYTHONPATH=/usr/local/lib/python3.12/site-packages:$PYTHONPATH" >> $GITHUB_ENV
- name: Run TTK tests
uses: ./.github/actions/test-ttk-unix
- uses: actions/checkout@v4
with:
repository: "topology-tool-kit/ttk-data"
ref: "dev"
path: "ttk-data"
name: Checkout ttk-data
- name: Run ttk-data states [NOT ENFORCED]
if: false
id: validate
continue-on-error: true
run: |
# remove buggy example
# related issue: https://github.com/topology-tool-kit/ttk/issues/1055
rm ttk-data/states/nestedTrackingFromOverlap.py
# remove examples which fill up the memory
rm ttk-data/states/topologicalOptimization_darkSky.pvsm
cd ttk-data/tests
mkdir output_screenshots
pvpython -u validate.py || (tar zcf screenshots.tar.gz output_screenshots && false)
env:
PV_PLUGIN_PATH: /usr/local/bin/plugins/TopologyToolKit
- name: Upload result screenshots
if: steps.validate.outcome == 'failure'
uses: actions/upload-artifact@v4
with:
name: screenshots-macOS.tar.gz
path: ttk-data/tests/screenshots.tar.gz
retention-days: 10
- name: Run ttk-data Python scripts
run: |
cd ttk-data
# remove buggy example
# related issue: https://github.com/topology-tool-kit/ttk/issues/1055
rm python/nestedTrackingFromOverlap.py
# tmp removal of mpi example
rm python/mpiExample.py
# remove examples which fill up the memory
rm python/topologicalOptimization_darkSky.py
pvpython -u python/run.py
env:
PV_PLUGIN_PATH: /usr/local/bin/plugins/TopologyToolKit
- name: Test ttk-data Python scripts results [NOT ENFORCED]
continue-on-error: true
run: |
cd ttk-data
cat python/res.json
diff python/hashes/macOS.json python/res.json
# ------------------ #
# Test Windows build #
# ------------------ #
test-build-windows:
runs-on: windows-2022
if: ${{ github.repository_owner == 'topology-tool-kit' || !contains(github.ref, 'heads') }}
env:
PV_DIR: C:\Program Files\TTK-ParaView
TTK_DIR: C:\Program Files (x86)\ttk
CONDA_ROOT: C:\Miniconda
steps:
- uses: actions/checkout@v4
name: Checkout TTK source code
- uses: s-weigand/setup-conda@v1
- name: Install dependencies with conda
shell: bash
run: |
conda install -c conda-forge libboost-devel glew eigen spectralib zfp \
sqlite scikit-learn graphviz ninja python=3.10 zlib qhull \
llvm-openmp clangxx=18
# add TTK & ParaView install folders to PATH
echo "$PV_DIR/bin" >> $GITHUB_PATH
echo "$TTK_DIR/bin" >> $GITHUB_PATH
conda info --all
- name: Install and setup sccache
uses: f3d-app/sccache-setup-action@v1
with:
key: windows-0
- name: Remove hosted Python
shell: bash
run: |
rm -rf C:/hostedtoolcache/windows/Python
- name: Fetch TTK-ParaView headless Windows installer
run: |
curl.exe -OL https://github.com/${{ env.PV_REPO }}/releases/download/${{ env.PV_TAG }}/ttk-paraview-headless.exe
- name: Install ParaView
shell: cmd
run: |
ttk-paraview-headless.exe /S
- name: Create & configure TTK build directory
shell: cmd
run: |
set CMAKE_PREFIX_PATH=%CONDA_ROOT%\Library\lib\cmake;%CONDA_ROOT%\Library\share\eigen3\cmake;%CONDA_ROOT%\Library\share\Qull\cmake;%CONDA_ROOT%\Library\cmake;%ProgramFiles%\TTK-ParaView\lib\cmake;
set CC=clang-cl.exe
set CXX=clang-cl.exe
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
mkdir build
cd build
cmake ^
-DCMAKE_BUILD_TYPE=Release ^
-DCMAKE_POLICY_DEFAULT_CMP0092=NEW ^
-DCMAKE_POLICY_DEFAULT_CMP0074=NEW ^
-DBUILD_SHARED_LIBS:BOOL=TRUE ^
-DCMAKE_C_COMPILER_LAUNCHER=sccache ^
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache ^
-DPython3_ROOT_DIR="%CONDA_ROOT%" ^
-DGraphviz_INCLUDE_DIR="%CONDA_ROOT%\Library\include\graphviz" ^
-DGraphviz_CDT_LIBRARY="%CONDA_ROOT%\Library\lib\cdt.lib" ^
-DGraphviz_GVC_LIBRARY="%CONDA_ROOT%\Library\lib\gvc.lib" ^
-DGraphviz_CGRAPH_LIBRARY="%CONDA_ROOT%\Library\lib\cgraph.lib" ^
-DGraphviz_PATHPLAN_LIBRARY="%CONDA_ROOT%\Library\lib\pathplan.lib" ^
-DZLIB_ROOT="%CONDA_ROOT%\Library" ^
-DSQLite3_ROOT="%CONDA_ROOT%\Library" ^
-DTTK_BUILD_PARAVIEW_PLUGINS=TRUE ^
-DTTK_BUILD_VTK_WRAPPERS=TRUE ^
-DTTK_BUILD_STANDALONE_APPS=TRUE ^
-DTTK_ENABLE_KAMIKAZE=TRUE ^
-DTTK_ENABLE_OPENMP=TRUE ^
-DTTK_ENABLE_CPU_OPTIMIZATION=FALSE ^
-DTTK_ENABLE_SHARED_BASE_LIBRARIES=TRUE ^
-DTTK_IMPLICIT_PRECONDITIONS_THRESHOLD=64*64*64 ^
-GNinja ^
..
env:
CXXFLAGS: -Werror
- name: Fix clang-cl OpenMP flags in build.ninja
shell: bash
run: |
sed -i 's/-Xclang -fopenmp/-openmp/' build/build.ninja
- name: Build & install TTK
shell: cmd
run: |
cd build
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
cmake --build . --config Release --parallel --target install
- name: Show sccache results
shell: bash
run: sccache --show-stats
- name: Test C++ example
shell: cmd
run: |
set CC=clang-cl.exe
set CXX=clang-cl.exe
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
set CMAKE_PREFIX_PATH=%TTK_DIR%\lib\cmake;%PV_DIR%\lib\cmake
cd %GITHUB_WORKSPACE%\examples\c++
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release -GNinja ..
cmake --build . --config Release --parallel
ttkExample-c++.exe -i ..\..\data\inputData.off
- name: Test VTK-C++ example
shell: cmd
run: |
set CC=clang-cl.exe
set CXX=clang-cl.exe
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
set CMAKE_PREFIX_PATH=%TTK_DIR%\lib\cmake;%PV_DIR%\lib\cmake
cd %GITHUB_WORKSPACE%\examples\vtk-c++
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release -GNinja ..
cmake --build . --config Release --parallel
ttkExample-vtk-c++.exe -i ..\..\data\inputData.vtu
- name: Test Python example [TEMPORARILY NOT ENFORCED]
continue-on-error: true
shell: cmd
run: |
set PYTHONPATH=%PV_DIR%\bin\Lib\site-packages;%TTK_DIR%\bin\Lib\site-packages;%CONDA_ROOT%\Lib
cd %GITHUB_WORKSPACE%\examples\python
python example.py ..\data\inputData.vtu
- name: Test pvpython example
shell: cmd
run: |
set PYTHONPATH=%CONDA_ROOT%\Lib
set PV_PLUGIN_PATH=%TTK_DIR%\bin\plugins
cd %GITHUB_WORKSPACE%\examples\pvpython
pvpython.exe example.py ..\data\inputData.vtu
- name: Test standalone
shell: cmd
run: |
cd %GITHUB_WORKSPACE%\examples\vtk-c++
ttkHelloWorldCmd.exe -i %GITHUB_WORKSPACE%\examples\data\inputData.vtu
- uses: actions/checkout@v4
with:
repository: "topology-tool-kit/ttk-data"
ref: "dev"
path: "ttk-data"
name: Checkout ttk-data
- name: Resample large datasets
shell: cmd
run: |
set PYTHONPATH=%CONDA_ROOT%\Lib
set PV_PLUGIN_PATH=%TTK_DIR%\bin\plugins
cd ttk-data
pvpython.exe python\resample.py ctBones.vti 128 128 128 ctBones.vti
- name: Run ttk-data Python scripts
shell: cmd
run: |
set PYTHONPATH=%PV_DIR%\bin\Lib\site-packages;%TTK_DIR%\bin\Lib\site-packages;%CONDA_ROOT%\Lib;%CONDA_ROOT%\DLLs
set PV_PLUGIN_PATH=%TTK_DIR%\bin\plugins
cd ttk-data
rm python/nestedTrackingFromOverlap.py
# tmp removal of mpi example
rm python/mpiExample.py
# remove examples which fill up the memory
rm python/topologicalOptimization_darkSky.py
pvpython.exe -u python\run.py
- name: Test ttk-data Python scripts results [NOT ENFORCED]
continue-on-error: true
shell: cmd
run: |
cd ttk-data
type python\res.json
FC python\hashes\windows.json python\res.json