Merge pull request #1923 from o1-labs/brian/pure-bindings

Build bindings in a nix derivation

.github/workflows/build-bindings.yml

@@ -27,10 +27,15 @@ jobs:
         run: |
           set -Eeu
           ./pin.sh
-          nix develop o1js --command bash -c "npm run build:update-bindings"
+          nix run o1js#update-bindings --max-jobs 4
+          #fail if this changes any files
+          cd src/bindings
+          git diff --exit-code
+      - name: add build to gc-root if on main
+        if: github.ref == 'refs/heads/main'
+        run: |
+          nix build o1js#o1js-bindings --out-link /home/app/actions-runner/nix-cache/main-bindings-gcroot
       - name: Cleanup the Nix store
         run: |
-          nix-env --delete-generations old
-          nix-collect-garbage -d --quiet
           nix-store --gc --print-dead
           nix-store --optimise

flake.nix

@@ -84,6 +84,42 @@
           { targets = ["wasm32-unknown-unknown" "x86_64-unknown-linux-gnu" ];
             extensions = [ "rust-src" ];
           });
+        rust-platform = pkgs.makeRustPlatform
+            { cargo = rust-channel;
+              rustc = rust-channel;
+            };
+
+        bindings-pkgs = with pkgs;
+            [ nodejs
+              nodePackages.npm
+              typescript
+              nodePackages.typescript-language-server
+
+              #Rustup doesn't allow local toolchains to contain 'nightly' in the name
+              #so the toolchain is linked with the name nix and rustup is wrapped in a shellscript
+              #which calls the nix toolchain instead of the nightly one
+              (writeShellApplication
+                { name = "rustup";
+                  text =
+                  ''
+                  if [ "$1" = run ] && { [ "$2" = nightly-2023-09-01 ] || [ "$2" = 1.72-x86_64-unknowl-linux-gnu ]; }
+                  then
+                    echo using nix toolchain
+                    ${rustup}/bin/rustup run nix "''${@:3}"
+                  else
+                    echo using plain rustup "$@"
+                    ${rustup}/bin/rustup "$@"
+                  fi
+                  '';
+                }
+              )
+              rustup
+              wasm-pack
+              binaryen # provides wasm-opt
+
+              dune_3
+            ] ++ commonOverrides.buildInputs ;
+
         inherit (nixpkgs) lib;
         # All the submodules required by .gitmodules
         submodules = map builtins.head (builtins.filter lib.isList
@@ -111,6 +147,56 @@
                 command "nix-shell"
               }.
             '';
+
+          o1js-npm-deps = pkgs.buildNpmPackage
+            { name = "o1js";
+              src = with pkgs.lib.fileset;
+                  (toSource {
+                    root = ./.;
+                    fileset = unions [
+                      ./package.json
+                      ./package-lock.json
+                    ];
+                  });
+              # If you see 'ERROR: npmDepsHash is out of date' in ci
+              # set this to blank run ``nix build o1js#o1js-bindings`
+              # If you don't want to install nix you can also set it to "" and run ci to get the new hash
+              # You should get an output like this:
+
+              # error: hash mismatch in fixed-output derivation '/nix/store/a03cg2az0b2cvjsp1wnr89clf31i79c1-o1js-npm-deps.drv':
+              # specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+              #    got:    sha256-8EPvXpOgn0nvm/pFKN3h6EMjabOeBqfy5optIfe8E8Q=
+              # replace npmDepsHash bellow with the new hash
+
+              npmDepsHash = "sha256-QLnSfX6JwYQXyHGNSxXdzqbhkbFl67sDrmlW/F6D/pw=";
+              # The prepack script runs the build script, which we'd rather do in the build phase.
+              npmPackFlags = [ "--ignore-scripts" ];
+              dontNpmBuild = true;
+              installPhase = ''
+                runHook preInstall
+
+                mkdir -p $out/lib
+                cp -r node_modules $out/lib
+
+                runHook postInstall
+              '';
+            };
+          test-vectors = rust-platform.buildRustPackage {
+            src = pkgs.lib.sourceByRegex ./src/mina/src
+              [
+                "^lib(/crypto(/proof-systems(/.*)?)?)?$"
+              ];
+            sourceRoot = "source/lib/crypto/proof-systems/poseidon/export_test_vectors";
+            patchPhase =
+            ''
+            cp ${./src/mina/src/lib/crypto/proof-systems/Cargo.lock} .
+            '';
+            name = "export_test_vectors";
+            version = "0.1.0";
+            cargoSha256 = "";
+            CARGO_TARGET_DIR = "./target";
+            cargoLock = { lockFile = ./src/mina/src/lib/crypto/proof-systems/Cargo.lock ; };
+          };
       in {
         formatter = pkgs.nixfmt;
         inherit mina;
@@ -124,48 +210,96 @@
             export RUSTUP_HOME
             rustup toolchain link nix ${rust-channel}
             '';
-            packages = with pkgs;
-              [ nodejs
-                nodePackages.npm
-                typescript
-                nodePackages.typescript-language-server
+            packages = bindings-pkgs;
+          });
 
-                #Rustup doesn't allow local toolchains to contain 'nightly' in the name
-                #so the toolchain is linked with the name nix and rustup is wrapped in a shellscript
-                #which calls the nix toolchain instead of the nightly one
-                (writeShellApplication
-                  { name = "rustup";
-                    text =
-                    ''
-                    if [ "$1" = run ] && [ "$2" = nightly-2023-09-01 ]
-                    then
-                      ${rustup}/bin/rustup run nix "''${@:3}"
-                    else
-                      ${rustup}/bin/rustup "$@"
-                    fi
-                    '';
-                  }
-                )
-                rustup
-                wasm-pack
-                binaryen # provides wasm-opt
 
-                dune_3
-              ] ++ commonOverrides.buildInputs ;
-          });
         };
         # TODO build from ./ocaml root, not ./. (after fixing a bug in dune-nix)
         packages = {
-          kim = pkgs.kimchi-rust-wasm;
           inherit dune-description;
-          bindings = prj.pkgs.o1js_bindings;
+          o1js-bindings = pkgs.stdenv.mkDerivation {
+            name = "o1js_bindings";
+            src = with pkgs.lib.fileset;
+            (toSource {
+              root = ./.;
+              fileset = unions [
+                ./src/mina
+                ./src/bindings/scripts
+                ./src/bindings/js
+                ./src/bindings/crypto
+                ./src/bindings/lib
+                ./src/bindings/mina-transaction/gen/dune
+                (fileFilter (file: file.hasExt "js") ./src/bindings/mina-transaction)
+                ./src/bindings/ocaml/lib
+                ./src/bindings/ocaml/dune
+                ./src/bindings/ocaml/dune-project
+                (fileFilter (file: file.hasExt "ml") ./src/bindings/ocaml)
+                ./package.json
+                ./package-lock.json
+                ./src/bindings/ocaml/jsoo_exports
+                ./dune-project
+                ./.prettierrc.cjs
+                ./src/build
+                ./src/snarky.d.ts
+              ];
+            });
+            inherit (inputs.mina.devShells."${system}".default)
+              PLONK_WASM_NODEJS
+              PLONK_WASM_WEB
+              MARLIN_PLONK_STUBS
+              ;
+            PREBUILT_KIMCHI_BINDINGS_JS_WEB =
+              "${mina.files.src-lib-crypto-kimchi_bindings-js-web}/src/lib/crypto/kimchi_bindings/js/web";
+            PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS =
+              "${mina.files.src-lib-crypto-kimchi_bindings-js-node_js}/src/lib/crypto/kimchi_bindings/js/node_js";
+            EXPORT_TEST_VECTORS = "${test-vectors}/bin/export_test_vectors";
+            buildInputs = bindings-pkgs ++ [ pkgs.bash ];
+            SKIP_MINA_COMMIT = true;
+            JUST_BINDINGS = true;
+            patchPhase = ''
+            patchShebangs ./src/bindings/scripts/
+            patchShebangs ./src/bindings/crypto/test-vectors/
+            '';
+            buildPhase =
+            ''
+            RUSTUP_HOME=$(pwd)/.rustup
+            export RUSTUP_HOME
+            rustup toolchain link nix ${rust-channel}
+            cp -r ${o1js-npm-deps}/lib/node_modules/ .
+
+            mkdir -p src/bindings/compiled/node_bindings
+            echo '// this file exists to prevent TS from type-checking `o1js_node.bc.cjs`' \
+              > src/bindings/compiled/node_bindings/o1js_node.bc.d.cts
+
+            npm run build:update-bindings
+
+            mkdir -p $out/mina-transaction
+            pushd ./src/bindings
+              rm -rf ./compiled/_node_bindings
+              cp -Lr ./compiled $out
+              cp -Lr ./mina-transaction/gen $out/mina-transaction/
+            popd
+            '';
+          };
+          kimchi = pkgs.kimchi-rust-wasm;
           ocaml-js = prj.pkgs.__ocaml-js__;
-          default = pkgs.buildNpmPackage
-            { name = "o1js";
-              src = ./.;
-              npmDepsHash = "sha256-++MTGDUVBccYN8LA2Xb0FkbrZ14ZyVCrDPESXa52AwQ=";
-              # TODO ideally re-build bindings here
-            };
+        };
+        apps = {
+          update-bindings = {
+            type = "app";
+            program = "${pkgs.writeShellApplication
+              { name = "update-bindings";
+                text =
+                ''
+                cp -r ${self.packages."${system}".o1js-bindings}/* ./src/bindings
+                chmod +w -R src/bindings/compiled
+                MINA_COMMIT=$(git -C src/mina rev-parse HEAD)
+                echo "The mina commit used to generate the backends for node and web is" "$MINA_COMMIT" \
+                  > src/bindings/MINA_COMMIT
+                '';
+              }}/bin/update-bindings";
+          };
         };
       });
 }

pin.sh

@@ -9,5 +9,6 @@ git submodule sync && git submodule update --init --recursive
 # Add the flake registry entry
 nix registry add o1js "git+file://$ROOT?submodules=1"
 # update mina input to local submodule
+# --override-input seems redundant but fixes a caching issue with local paths
 nix flake update mina --override-input mina 'path:src/mina' --flake '.?submodules=1'
 popd