implement BridgeStan download and module compilation on Rust

.github/workflows/main.yaml

@@ -3,7 +3,7 @@ name: bridgestan tests
 on:
   push:
     branches:
-      - 'main'
+      - "main"
   pull_request:
   workflow_dispatch: {}
 
@@ -128,7 +128,6 @@ jobs:
         env:
           BRIDGESTAN: ${{ github.workspace }}
 
-
   julia:
     needs: [build]
     runs-on: ${{matrix.os}}
@@ -270,12 +269,6 @@ jobs:
           path: ./test_models/
           key: ${{ hashFiles('**/*.stan', 'src/*', 'stan/src/stan/version.hpp', 'Makefile') }}-${{ matrix.os }}-v${{ env.CACHE_VERSION }}
 
-      - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v2
-        with:
-          version: "15.0"
-          directory: ${{ runner.temp }}/llvm
-
       - name: Set up TBB
         if: matrix.os == 'windows-latest'
         run: |
@@ -284,11 +277,16 @@ jobs:
       - name: Run rust tests
         working-directory: ./rust
         timeout-minutes: 60
-        env:
-          LIBCLANG_PATH: ${{ runner.temp }}/llvm/lib
-          LLVM_CONFIG_PATH: ${{ runner.temp }}/llvm/bin/llvm-config
         run: |
           cargo clippy
           cargo fmt --check
-          cargo run --example=example
+          cargo run --example=example --features download-bridgestan-src
+
+          # run all tests except model_downloading
           cargo test --verbose
+          cargo test --verbose model_compiling -- --ignored
+
+          # run all tests with feature download-bridgestan-src
+          cargo test --verbose --features download-bridgestan-src
+          cargo test --verbose --features download-bridgestan-src model_downloading -- --ignored
+          cargo test --verbose --features download-bridgestan-src model_compiling -- --ignored

.gitignore

@@ -12,6 +12,7 @@ c-example/example_static
 # Rust
 rust/target/
 rust/Cargo.lock
+rust/.vscode
 
 notes.org
 

docs/languages/rust.rst

@@ -14,13 +14,9 @@ The BridgeStan Rust client is available on `crates.io <https://crates.io/crates/
 
     cargo add bridgestan
 
-To build and use BridgeStan models, a copy of the BridgeStan C++ source code
-is required. Please follow the :doc:`Getting Started guide <../getting-started>`
-or use the Rust client in tandem with an interface such as :doc:`Python <./python>`
-which automates this process.
+The first time you compile a model, the BridgeStan source code will be downloaded to `~/.bridgestan`. If you prefer to use a source distribution of BridgeStan, you can pass its path as the `bs_path` argument to `compile_model`.
 
-``STAN_THREADS=true`` needs to be specified when compiling a model, for more
-details see the `API reference <https://docs.rs/bridgestan>`__.
+Note that the system pre-requisites from the [Getting Started Guide](../getting-started.rst) are still required and will not be automatically installed by this method.
 
 Example Program
 ---------------

rust/Cargo.toml

@@ -11,10 +11,24 @@ homepage = "https://roualdes.github.io/bridgestan/latest/"
 [dependencies]
 libloading = "0.8.0"
 thiserror = "1.0.40"
+ureq = { version = "2.7", optional = true }
+tar = { version = "0.4", optional = true }
+flate2 = { version = "1.0", optional = true }
+dirs = { version = "5.0", optional = true }
+path-absolutize = { version = "3.1" }
+log = { version = "0.4" }
+
+[features]
+download-bridgestan-src = ["dep:ureq", "dep:tar", "dep:flate2", "dep:dirs"]
 
 [build-dependencies]
 bindgen = "0.69.1"
 
 [dev-dependencies]
 approx = "0.5.1"
 rand = "0.8.5"
+env_logger = "0.11"
+dirs = { version = "5.0" }
+
+[[example]]
+name = "example"

rust/README.md

@@ -11,11 +11,8 @@ Internally, it relies on [`bindgen`](https://docs.rs/bindgen/) and
 
 ## Compiling the model
 
-The Rust wrapper does not currently have any functionality to compile Stan models.
-Compiled shared libraries need to be built manually using `make` or with the Julia
-or Python bindings.
+The Rust wrapper currently have a builtin functionality to compile Stan models (function `compile_model` under the feature flag `download-bridgestan-src`). For safety reasons all Stan models need to be installed with `STAN_THREADS=true`. However, if you use the Rust wrapper builtin functionality to compile Stan models, this will automatically be set for you.
 
-For safety reasons all Stan models need to be installed with `STAN_THREADS=true`.
 When compiling a model using `make`, set the environment variable:
 
 ```bash
@@ -32,21 +29,30 @@ path = bridgestan.compile_model("stan_model.stan", make_args=["STAN_THREADS=true
 If `STAN_THREADS` was not specified while building the model, the Rust wrapper
 will throw an error when loading the model.
 
-## Usage:
+## Usage
 
-Run this example with `cargo run --example=example`.
+Run this example with `RUST_LOG=info cargo run --example=example --features download-bridgestan-src`.
 
 ```rust
 use std::ffi::CString;
-use std::path::Path;
-use bridgestan::{BridgeStanError, Model, open_library};
+use std::path::{Path, PathBuf};
+use bridgestan::{BridgeStanError, Model, open_library, compile_model};
 
-// The path to the compiled model.
-// Get for instance from python `bridgestan.compile_model`
+// The path to the Stan model
 let path = Path::new(env!["CARGO_MANIFEST_DIR"])
     .parent()
     .unwrap()
-    .join("test_models/simple/simple_model.so");
+    .join("test_models/simple/simple.stan");
+
+// You can manually set the BridgeStan src path or
+// automatically download it (but remember to
+// enable the download-bridgestan-src feature first)
+let bs_path: PathBuf = "..".into();
+// let bs_path = bridgestan::download_bridgestan_src().unwrap();
+
+// The path to the compiled model
+let path = compile_model(&bs_path, &path, vec![], vec![]).expect("Could not compile Stan model.");
+println!("Compiled model: {:?}", path);
 
 let lib = open_library(path).expect("Could not load compiled Stan model.");
 

rust/examples/example.rs

@@ -1,14 +1,31 @@
-use bridgestan::{open_library, BridgeStanError, Model};
+use bridgestan::{compile_model, open_library, BridgeStanError, Model};
 use std::ffi::CString;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 fn main() {
-    // The path to the compiled model.
-    // Get for instance from python `bridgestan.compile_model`
+    if std::env::var("RUST_LOG").is_err() {
+        std::env::set_var("RUST_LOG", "bridgestan=info");
+    }
+    env_logger::init();
+
+    // The path to the Stan model
     let path = Path::new(env!["CARGO_MANIFEST_DIR"])
         .parent()
         .unwrap()
-        .join("test_models/simple/simple_model.so");
+        .join("test_models")
+        .join("simple")
+        .join("simple.stan");
+
+    // You can manually set the BridgeStan src path or
+    // automatically download it (but remember to
+    // enable the download-bridgestan-src feature first)
+    let bs_path: PathBuf = "..".into();
+    // let bs_path = bridgestan::download_bridgestan_src().unwrap();
+
+    // The path to the compiled model
+    let path =
+        compile_model(&bs_path, &path, vec![], vec![]).expect("Could not compile Stan model.");
+    println!("Compiled model: {:?}", path);
 
     let lib = open_library(path).expect("Could not load compiled Stan model.");
 

rust/src/bs_safe.rs

@@ -1,4 +1,5 @@
 use crate::ffi;
+use crate::VERSION;
 use std::borrow::Borrow;
 use std::collections::hash_map::DefaultHasher;
 use std::ffi::c_char;
@@ -101,9 +102,15 @@ pub enum BridgeStanError {
     /// Setting a print-callback failed.
     #[error("Failed to set a print-callback: {0}")]
     SetCallbackFailed(String),
+    /// Setting a compile Stan model failed.
+    #[error("Failed to compile Stan model: {0}")]
+    ModelCompilingFailed(String),
+    /// Setting a download BridgeStan failed.
+    #[error("Failed to download BridgeStan {VERSION} from github.com: {0}")]
+    DownloadFailed(String),
 }
 
-type Result<T> = std::result::Result<T, BridgeStanError>;
+pub(crate) type Result<T> = std::result::Result<T, BridgeStanError>;
 
 /// Open a compiled Stan library.
 ///

rust/src/compile.rs

@@ -0,0 +1,90 @@
+use crate::bs_safe::{BridgeStanError, Result};
+use log::info;
+use path_absolutize::Absolutize;
+use std::path::{Path, PathBuf};
+
+/// Compile a Stan Model given the path to BridgeStan and to a stan_file
+pub fn compile_model(
+    bs_path: &Path,
+    stan_file: &Path,
+    stanc_args: Vec<&str>,
+    make_args: Vec<&str>,
+) -> Result<PathBuf> {
+    // using path_absolutize crate for now since
+    // std::fs::canonicalize doesn't behave well on windows
+    // we may switch to std::path::absolute once it stabilizes, see
+    // https://github.com/roualdes/bridgestan/pull/212#discussion_r1513375667
+    let stan_file = stan_file
+        .absolutize()
+        .map_err(|e| BridgeStanError::ModelCompilingFailed(e.to_string()))?;
+
+    // get --include-paths=model_dir
+    let includir_stan_file_dir = stan_file
+        .parent()
+        .and_then(Path::to_str)
+        .map(|x| format!("--include-paths={x}"))
+        .map(|x| vec![x])
+        .unwrap_or_default();
+    let includir_stan_file_dir = includir_stan_file_dir
+        .iter()
+        .map(String::as_str)
+        .collect::<Vec<&str>>();
+
+    if stan_file.extension().unwrap_or_default() != "stan" {
+        return Err(BridgeStanError::ModelCompilingFailed(
+            "File must be a .stan file".to_owned(),
+        ));
+    }
+
+    // add _model suffix and change extension to .so
+    let output = stan_file.with_extension("");
+    let output = output.with_file_name(format!(
+        "{}_model",
+        output.file_name().unwrap_or_default().to_string_lossy()
+    ));
+    let output = output.with_extension("so");
+
+    let stanc_args = [includir_stan_file_dir.as_slice(), stanc_args.as_slice()].concat();
+    let stanc_args = stanc_args.join(" ");
+    let stanc_args = format!("STANCFLAGS={}", stanc_args);
+    let stanc_args = [stanc_args.as_str()];
+
+    let cmd = [
+        &[output.to_str().unwrap_or_default()],
+        make_args.as_slice(),
+        stanc_args.as_slice(),
+    ]
+    .concat();
+
+    let make = if cfg!(target_os = "windows") {
+        "mingw32-make"
+    } else {
+        "make"
+    };
+    info!(
+        "Compiling model with command: {} \"{}\"",
+        make,
+        cmd.join("\" \"")
+    );
+    std::process::Command::new(make)
+        .args(cmd)
+        .current_dir(bs_path)
+        .env("STAN_THREADS", "true")
+        .output()
+        .map_err(|e| e.to_string())
+        .and_then(|proc| {
+            if !proc.status.success() {
+                Err(format!(
+                    "{} {}",
+                    String::from_utf8_lossy(proc.stdout.as_slice()).into_owned(),
+                    String::from_utf8_lossy(proc.stderr.as_slice()).into_owned(),
+                ))
+            } else {
+                Ok(())
+            }
+        })
+        .map_err(|e| BridgeStanError::ModelCompilingFailed(e.to_string()))?;
+    info!("Finished compiling model");
+
+    Ok(output)
+}

rust/src/download.rs

@@ -0,0 +1,62 @@
+use crate::bs_safe::{BridgeStanError, Result};
+use crate::VERSION;
+use flate2::read::GzDecoder;
+use log::info;
+use std::{env::temp_dir, fs, path::PathBuf};
+use tar::Archive;
+
+/// Download and unzip the BridgeStan source distribution for this version
+/// to ~/.bridgestan/bridgestan-version
+pub fn download_bridgestan_src() -> Result<PathBuf> {
+    let homedir = dirs::home_dir().unwrap_or(temp_dir());
+
+    let bs_path_download_temp = homedir.join(".bridgestan_tmp_dir");
+    let bs_path_download = homedir.join(".bridgestan");
+
+    let bs_path_download_temp_join_version =
+        bs_path_download_temp.join(format!("bridgestan-{VERSION}"));
+    let bs_path_download_join_version = bs_path_download.join(format!("bridgestan-{VERSION}"));
+
+    if !bs_path_download_join_version.exists() {
+        info!("Downloading BridgeStan");
+
+        fs::remove_dir_all(&bs_path_download_temp).unwrap_or_default();
+        fs::create_dir(&bs_path_download_temp).unwrap_or_default();
+        fs::create_dir(&bs_path_download).unwrap_or_default();
+
+        let url = "https://github.com/roualdes/bridgestan/releases/download/".to_owned()
+            + format!("v{VERSION}/bridgestan-{VERSION}.tar.gz").as_str();
+
+        let response = ureq::get(url.as_str())
+            .call()
+            .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+        let len = response
+            .header("Content-Length")
+            .and_then(|s| s.parse::<usize>().ok())
+            .unwrap_or(50_000_000);
+
+        let mut bytes: Vec<u8> = Vec::with_capacity(len);
+        response
+            .into_reader()
+            .read_to_end(&mut bytes)
+            .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+
+        let tar = GzDecoder::new(bytes.as_slice());
+        let mut archive = Archive::new(tar);
+        archive
+            .unpack(&bs_path_download_temp)
+            .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+
+        fs::rename(
+            bs_path_download_temp_join_version,
+            &bs_path_download_join_version,
+        )
+        .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+
+        fs::remove_dir(bs_path_download_temp).unwrap_or_default();
+
+        info!("Finished downloading BridgeStan");
+    }
+
+    Ok(bs_path_download_join_version)
+}

rust/src/lib.rs

@@ -1,6 +1,17 @@
 #![doc = include_str!("../README.md")]
 
 mod bs_safe;
-pub(crate) mod ffi;
+mod compile;
+
+#[cfg(feature = "download-bridgestan-src")]
+mod download;
 
+pub(crate) mod ffi;
 pub use bs_safe::{open_library, BridgeStanError, Model, Rng, StanLibrary};
+
+#[cfg(feature = "download-bridgestan-src")]
+pub use download::download_bridgestan_src;
+
+pub use compile::compile_model;
+
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");