diff --git a/.github/workflows/build-bindings.yml b/.github/workflows/build-bindings.yml index d1b6772a0..38e8f821e 100644 --- a/.github/workflows/build-bindings.yml +++ b/.github/workflows/build-bindings.yml @@ -27,10 +27,15 @@ jobs: run: | set -Eeu ./pin.sh - nix develop o1js --command bash -c "npm run build:update-bindings" + nix run o1js#update-bindings --max-jobs 4 + #fail if this changes any files + cd src/bindings + git diff --exit-code + - name: add build to gc-root if on main + if: github.ref == 'refs/heads/main' + run: | + nix build o1js#o1js-bindings --out-link /home/app/actions-runner/nix-cache/main-bindings-gcroot - name: Cleanup the Nix store run: | - nix-env --delete-generations old - nix-collect-garbage -d --quiet nix-store --gc --print-dead nix-store --optimise diff --git a/flake.lock b/flake.lock index 79916af26..49061b354 100644 --- a/flake.lock +++ b/flake.lock @@ -265,8 +265,8 @@ "utils": "utils" }, "locked": { - "lastModified": 1731611028, - "narHash": "sha256-sehBPO+H9mtShGb3KNhdDVHj0pogt/tmiL9tc2ICkaU=", + "lastModified": 1733429866, + "narHash": "sha256-/ZEGYdZ2hLjBwdEzG/BIjlDehOjuGWmBqzD55nXfZoY=", "path": "src/mina", "type": "path" }, diff --git a/flake.nix b/flake.nix index dd7e4d3d4..74f10868a 100644 --- a/flake.nix +++ b/flake.nix @@ -84,6 +84,42 @@ { targets = ["wasm32-unknown-unknown" "x86_64-unknown-linux-gnu" ]; extensions = [ "rust-src" ]; }); + rust-platform = pkgs.makeRustPlatform + { cargo = rust-channel; + rustc = rust-channel; + }; + + bindings-pkgs = with pkgs; + [ nodejs + nodePackages.npm + typescript + nodePackages.typescript-language-server + + #Rustup doesn't allow local toolchains to contain 'nightly' in the name + #so the toolchain is linked with the name nix and rustup is wrapped in a shellscript + #which calls the nix toolchain instead of the nightly one + (writeShellApplication + { name = "rustup"; + text = + '' + if [ "$1" = run ] && { [ "$2" = nightly-2023-09-01 ] || [ "$2" = 1.72-x86_64-unknowl-linux-gnu ]; } + then + echo using nix toolchain + ${rustup}/bin/rustup run nix "''${@:3}" + else + echo using plain rustup "$@" + ${rustup}/bin/rustup "$@" + fi + ''; + } + ) + rustup + wasm-pack + binaryen # provides wasm-opt + + dune_3 + ] ++ commonOverrides.buildInputs ; + inherit (nixpkgs) lib; # All the submodules required by .gitmodules submodules = map builtins.head (builtins.filter lib.isList @@ -111,6 +147,56 @@ command "nix-shell" }. ''; + + o1js-npm-deps = pkgs.buildNpmPackage + { name = "o1js"; + src = with pkgs.lib.fileset; + (toSource { + root = ./.; + fileset = unions [ + ./package.json + ./package-lock.json + ]; + }); + # If you see 'ERROR: npmDepsHash is out of date' in ci + # set this to blank run ``nix build o1js#o1js-bindings` + # If you don't want to install nix you can also set it to "" and run ci to get the new hash + # You should get an output like this: + + # error: hash mismatch in fixed-output derivation '/nix/store/a03cg2az0b2cvjsp1wnr89clf31i79c1-o1js-npm-deps.drv': + # specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + # got: sha256-8EPvXpOgn0nvm/pFKN3h6EMjabOeBqfy5optIfe8E8Q= + # replace npmDepsHash bellow with the new hash + + npmDepsHash = "sha256-QLnSfX6JwYQXyHGNSxXdzqbhkbFl67sDrmlW/F6D/pw="; + # The prepack script runs the build script, which we'd rather do in the build phase. + npmPackFlags = [ "--ignore-scripts" ]; + dontNpmBuild = true; + installPhase = '' + runHook preInstall + + mkdir -p $out/lib + cp -r node_modules $out/lib + + runHook postInstall + ''; + }; + test-vectors = rust-platform.buildRustPackage { + src = pkgs.lib.sourceByRegex ./src/mina/src + [ + "^lib(/crypto(/proof-systems(/.*)?)?)?$" + ]; + sourceRoot = "source/lib/crypto/proof-systems/poseidon/export_test_vectors"; + patchPhase = + '' + cp ${./src/mina/src/lib/crypto/proof-systems/Cargo.lock} . + ''; + name = "export_test_vectors"; + version = "0.1.0"; + cargoSha256 = ""; + CARGO_TARGET_DIR = "./target"; + cargoLock = { lockFile = ./src/mina/src/lib/crypto/proof-systems/Cargo.lock ; }; + }; in { formatter = pkgs.nixfmt; inherit mina; @@ -124,48 +210,96 @@ export RUSTUP_HOME rustup toolchain link nix ${rust-channel} ''; - packages = with pkgs; - [ nodejs - nodePackages.npm - typescript - nodePackages.typescript-language-server + packages = bindings-pkgs; + }); - #Rustup doesn't allow local toolchains to contain 'nightly' in the name - #so the toolchain is linked with the name nix and rustup is wrapped in a shellscript - #which calls the nix toolchain instead of the nightly one - (writeShellApplication - { name = "rustup"; - text = - '' - if [ "$1" = run ] && [ "$2" = nightly-2023-09-01 ] - then - ${rustup}/bin/rustup run nix "''${@:3}" - else - ${rustup}/bin/rustup "$@" - fi - ''; - } - ) - rustup - wasm-pack - binaryen # provides wasm-opt - dune_3 - ] ++ commonOverrides.buildInputs ; - }); }; # TODO build from ./ocaml root, not ./. (after fixing a bug in dune-nix) packages = { - kim = pkgs.kimchi-rust-wasm; inherit dune-description; - bindings = prj.pkgs.o1js_bindings; + o1js-bindings = pkgs.stdenv.mkDerivation { + name = "o1js_bindings"; + src = with pkgs.lib.fileset; + (toSource { + root = ./.; + fileset = unions [ + ./src/mina + ./src/bindings/scripts + ./src/bindings/js + ./src/bindings/crypto + ./src/bindings/lib + ./src/bindings/mina-transaction/gen/dune + (fileFilter (file: file.hasExt "js") ./src/bindings/mina-transaction) + ./src/bindings/ocaml/lib + ./src/bindings/ocaml/dune + ./src/bindings/ocaml/dune-project + (fileFilter (file: file.hasExt "ml") ./src/bindings/ocaml) + ./package.json + ./package-lock.json + ./src/bindings/ocaml/jsoo_exports + ./dune-project + ./.prettierrc.cjs + ./src/build + ./src/snarky.d.ts + ]; + }); + inherit (inputs.mina.devShells."${system}".default) + PLONK_WASM_NODEJS + PLONK_WASM_WEB + MARLIN_PLONK_STUBS + ; + PREBUILT_KIMCHI_BINDINGS_JS_WEB = + "${mina.files.src-lib-crypto-kimchi_bindings-js-web}/src/lib/crypto/kimchi_bindings/js/web"; + PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS = + "${mina.files.src-lib-crypto-kimchi_bindings-js-node_js}/src/lib/crypto/kimchi_bindings/js/node_js"; + EXPORT_TEST_VECTORS = "${test-vectors}/bin/export_test_vectors"; + buildInputs = bindings-pkgs ++ [ pkgs.bash ]; + SKIP_MINA_COMMIT = true; + JUST_BINDINGS = true; + patchPhase = '' + patchShebangs ./src/bindings/scripts/ + patchShebangs ./src/bindings/crypto/test-vectors/ + ''; + buildPhase = + '' + RUSTUP_HOME=$(pwd)/.rustup + export RUSTUP_HOME + rustup toolchain link nix ${rust-channel} + cp -r ${o1js-npm-deps}/lib/node_modules/ . + + mkdir -p src/bindings/compiled/node_bindings + echo '// this file exists to prevent TS from type-checking `o1js_node.bc.cjs`' \ + > src/bindings/compiled/node_bindings/o1js_node.bc.d.cts + + npm run build:update-bindings + + mkdir -p $out/mina-transaction + pushd ./src/bindings + rm -rf ./compiled/_node_bindings + cp -Lr ./compiled $out + cp -Lr ./mina-transaction/gen $out/mina-transaction/ + popd + ''; + }; + kimchi = pkgs.kimchi-rust-wasm; ocaml-js = prj.pkgs.__ocaml-js__; - default = pkgs.buildNpmPackage - { name = "o1js"; - src = ./.; - npmDepsHash = "sha256-++MTGDUVBccYN8LA2Xb0FkbrZ14ZyVCrDPESXa52AwQ="; - # TODO ideally re-build bindings here - }; + }; + apps = { + update-bindings = { + type = "app"; + program = "${pkgs.writeShellApplication + { name = "update-bindings"; + text = + '' + cp -r ${self.packages."${system}".o1js-bindings}/* ./src/bindings + chmod +w -R src/bindings/compiled + MINA_COMMIT=$(git -C src/mina rev-parse HEAD) + echo "The mina commit used to generate the backends for node and web is" "$MINA_COMMIT" \ + > src/bindings/MINA_COMMIT + ''; + }}/bin/update-bindings"; + }; }; }); } diff --git a/pin.sh b/pin.sh index 6815dbac8..4007b0d18 100755 --- a/pin.sh +++ b/pin.sh @@ -9,5 +9,6 @@ git submodule sync && git submodule update --init --recursive # Add the flake registry entry nix registry add o1js "git+file://$ROOT?submodules=1" # update mina input to local submodule +# --override-input seems redundant but fixes a caching issue with local paths nix flake update mina --override-input mina 'path:src/mina' --flake '.?submodules=1' popd diff --git a/src/bindings b/src/bindings index 34a05bc88..a5a0883f0 160000 --- a/src/bindings +++ b/src/bindings @@ -1 +1 @@ -Subproject commit 34a05bc8888eb577aa73852dab5df90c7cae1050 +Subproject commit a5a0883f033afd4c9fd4bf96a5e0b8ebd92a87c9