From 4b3a198904d2ed55e1753557beb246fc214443e2 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Tue, 15 Mar 2022 03:37:36 +0100
Subject: [PATCH] Add support for patching dependencies

Can be used to patchShebangs for specific packages, which with NPM
version 7.0 can't be done globally for all packages anymore

Could also be a replacement for preInstallLinks
---
 API.md                                        |  21 +++
 internal.nix                                  | 147 ++++++++++++++++--
 .../source-patching/package-lock.json         |  13 ++
 .../source-patching/package.json              |  11 ++
 .../source-patching/shell.nix                 |  25 +++
 tests/integration-tests/default.nix           |   8 +
 tests/lib.nix                                 |   5 +-
 tests/make-source.nix                         |   5 +-
 8 files changed, 217 insertions(+), 18 deletions(-)
 create mode 100644 tests/examples-projects/source-patching/package-lock.json
 create mode 100644 tests/examples-projects/source-patching/package.json
 create mode 100644 tests/examples-projects/source-patching/shell.nix

diff --git a/API.md b/API.md
index ea50b97..8d1e70c 100644
--- a/API.md
+++ b/API.md
@@ -16,6 +16,7 @@ The `node_modules` function takes an attribute set with the following attributes
 - **nodejs** *(default `nixpkgs.nodejs`, which is the Active LTS version)*: Node.js derivation to use
 - **preInstallLinks** *(default `{}`)*: Map of symlinks to create inside npm dependencies in the `node_modules` output (See [Concepts](#concepts) for details).
 - **githubSourceHashMap** *(default `{}`)*: Dependency hashes for evaluation in restricted mode (See [Concepts](#concepts) for details).
+- **sourceMods** *(default `{}`)*: Source modifications (See [Concepts](#concepts) for details)
 
 #### Notes
 - You may provide additional arguments accepted by `mkDerivation` all of which are going to be passed on.
@@ -114,3 +115,23 @@ npmlock2nix.build {
   };
 }
 ```
+
+### Source modifications
+
+`node_modules` takes a `sourceMods` argument, which allows you to modify sources of individual npm packages you depend on, useful for adding Nix-specific fixes to packages. This could be used for patching interpreter or paths, or to replace vendored binaries with ones provided by Nix:
+
+```nix
+npmlock2nix.node_modules {
+  sourceMods = {
+    # <package name> = { version }: {
+    #   patches = [ <patch> ];
+    #   postPatch = <script>;
+    # }
+
+    # Patching shebangs
+    node-pre-gyp = _: {
+      postPatch = "patchShebangs bin";
+    };
+  };
+}
+```
diff --git a/internal.nix b/internal.nix
index 4bd9c16..dbf8c0c 100644
--- a/internal.nix
+++ b/internal.nix
@@ -1,4 +1,4 @@
-{ nodejs, stdenv, mkShell, lib, fetchurl, writeText, writeTextFile, runCommand, fetchFromGitHub }:
+{ nodejs, jq, openssl, stdenv, mkShell, lib, fetchurl, writeText, writeTextFile, runCommand, fetchFromGitHub }:
 rec {
   default_nodejs = nodejs;
 
@@ -60,7 +60,7 @@ rec {
   # hash attribute it will provide the value to `fetchFromGitHub` which will
   # also work in restricted evaluation.
   # Type: Set -> Path
-  buildTgzFromGitHub = { name, org, repo, rev, ref, hash ? null }:
+  buildTgzFromGitHub = { name, org, repo, rev, ref, hash ? null, sourceOptions ? { } }:
     let
       src =
         if hash != null then
@@ -76,20 +76,43 @@ rec {
             inherit rev ref;
             allRefs = true;
           };
+
+      # If sourceMods.${name} is set, call it with some information about this
+      # source for it to make a decision about when to do modifications
+      mods = lib.optionalAttrs (sourceOptions ? sourceMods.${name})
+        (sourceOptions.sourceMods.${name} ({
+          version = ref;
+        }));
     in
-    runCommand
-      name
-      { } ''
-      set +x
-      tar -C ${src} -czf $out ./
+    packTgz sourceOptions.nodejs name ref src mods;
+
+  # Description: Packs a source directory into a .tgz tar archive, allowing the
+  # source to be modified using derivation attributes patches and
+  # postPatch. If the source is an archive, it gets unpacked first.
+  # Type: Path -> String -> Path -> Set -> Path
+  packTgz = nodejs: pname: version: src: { patches ? [ ], postPatch ? "" }: stdenv.mkDerivation {
+    name = "${pname}-${version}.tgz";
+    inherit src;
+    nativeBuildInputs = attrs.nativeBuildInputs or [ ] ++ [
+      # Allows patchShebangs in postPatch to patch shebangs to nodejs
+      nodejs
+    ];
+    phases = "unpackPhase patchPhase installPhase";
+    inherit patches postPatch;
+    installPhase = ''
+      runHook preInstall
+      tar -C . -czf $out ./
+      runHook postInstall
     '';
+  };
 
   # Description: Turns a dependency with a from field of the format
   # `github:org/repo#revision` into a git fetcher. The fetcher can
   # receive a hash value by calling 'sourceHashFunc' if a source hash
-  # map has been provided. Otherwise the function yields `null`.
+  # map has been provided. Otherwise the function yields `null`. Patches
+  # specified with sourceMods will be applied
   # Type: { sourceHashFunc :: Fn } -> String -> Set -> Path
-  makeGithubSource = { sourceHashFunc, ... }: name: dependency:
+  makeGithubSource = sourceOptions@{ sourceHashFunc, ... }: name: dependency:
     assert !(dependency ? version) ->
       builtins.throw "version` attribute missing from `${name}`";
     assert (lib.hasPrefix "github: " dependency.version) -> builtins.throw "invalid prefix for `version` field of `${name}` expected `github:`, got: `${dependency.version}`.";
@@ -106,6 +129,7 @@ rec {
         ref = f.rev;
         inherit (v) org repo rev;
         hash = sourceHashFunc { type = "github"; value = v; };
+        inherit sourceOptions;
       };
     in
     (builtins.removeAttrs dependency [ "from" ]) // {
@@ -125,6 +149,33 @@ rec {
   shouldUseVersionAsUrl = dependency:
     dependency ? version && dependency ? integrity && ! (dependency ? resolved) && looksLikeUrl dependency.version;
 
+  # Description: Replaces the `resolved` field of a dependency with a
+  # prefetched version from the Nix store. Patches specified with sourceMods
+  # will be applied, in which case the `integrity` attribute is set to `null`,
+  # in order to be recomputer later
+  # Type: { sourceMods :: Fn, nodejs :: Package } -> String -> Set -> Set
+  makeUrlSource = { sourceMods ? { }, nodejs, ... }: name: dependency:
+    let
+      src = fetchurl (makeSourceAttrs name dependency);
+      mods = sourceMods.${name} {
+        # Pass the version to the modification function so it can base its
+        # decision to make a modification based on it
+        inherit (dependency) version;
+      };
+      tgz =
+        if sourceMods ? ${name}
+        # If we have modification to this source, unpack the tgz, apply the
+        # patches and repack the tgz
+        then packTgz nodejs name dependency.version src mods
+        else src;
+      resolved = "file://" + toString tgz;
+    in
+    dependency // { inherit resolved; } // lib.optionalAttrs (sourceMods ? ${name}) {
+      # Integrity was tampered with due to the source mods, so it needs to be
+      # recalculated, which is done in the node_modules builder
+      integrity = null;
+    };
+
   # Description: Turns an npm lockfile dependency into a fetchurl derivation
   # Type: { sourceHashFunc :: Fn } -> String -> Set -> Derivation
   makeSource = sourceOptions: name: dependency:
@@ -133,7 +184,7 @@ rec {
     assert (builtins.typeOf dependency != "set") ->
       throw "Specification of dependency ${toString name} must be a set";
     if dependency ? resolved && dependency ? integrity then
-      dependency // { resolved = "file://" + (toString (fetchurl (makeSourceAttrs name dependency))); }
+      makeUrlSource sourceOptions name dependency
     else if dependency ? from && dependency ? version then
       makeGithubSource sourceOptions name dependency
     else if shouldUseVersionAsUrl dependency then
@@ -156,7 +207,7 @@ rec {
 
   # Description: Turns a github string reference into a store path with a tgz of the reference
   # Type: Fn -> String -> String -> Path
-  stringToTgzPath = { sourceHashFunc, ... }: name: str:
+  stringToTgzPath = sourceOptions@{ sourceHashFunc, ... }: name: str:
     let
       gitAttrs = parseGitHubRef str;
     in
@@ -165,6 +216,7 @@ rec {
       ref = gitAttrs.rev;
       inherit (gitAttrs) org repo rev;
       hash = sourceHashFunc { type = "github"; value = gitAttrs; };
+      inherit sourceOptions;
     };
 
   # Description: Patch the `requires` attributes of a dependency spec to refer to paths in the store
@@ -314,6 +366,7 @@ rec {
     , preBuild ? ""
     , postBuild ? ""
     , preInstallLinks ? { } # set that describes which files should be linked in a specific packages folder
+    , sourceMods ? { }
     , githubSourceHashMap ? { }
     , passthru ? { }
     , ...
@@ -321,13 +374,17 @@ rec {
       assert (builtins.typeOf preInstallLinks != "set") ->
         throw "`preInstallLinks` must be an attributeset of attributesets";
       let
-        cleanArgs = builtins.removeAttrs args [ "src" "packageJson" "packageLockJson" "buildInputs" "nativeBuildInputs" "nodejs" "preBuild" "postBuild" "preInstallLinks" "githubSourceHashMap" ];
+        cleanArgs = builtins.removeAttrs args [ "src" "packageJson" "packageLockJson" "buildInputs" "nativeBuildInputs" "nodejs" "preBuild" "postBuild" "preInstallLinks" "sourceMods" "githubSourceHashMap" ];
         lockfile = readLockfile packageLockJson;
 
         sourceOptions = {
           sourceHashFunc = sourceHashFunc githubSourceHashMap;
+          inherit nodejs sourceMods;
         };
 
+        patchedLockfilePath = patchedLockfile sourceOptions packageLockJson;
+        patchedPackagefilePath = patchedPackagefile packageJson;
+
         preinstall_node_modules = writeTextFile {
           name = "prepare";
           destination = "/node_modules/.hooks/prepare";
@@ -372,6 +429,8 @@ rec {
         dontUnpack = true;
 
         nativeBuildInputs = nativeBuildInputs ++ [
+          jq
+          openssl
           nodejs
         ];
 
@@ -387,9 +446,65 @@ rec {
           export HOME=$(mktemp -d)
         '';
 
+        # A jq filter for finding dependencies with an integrity field of
+        # `null`, as set at evaluation time by `makeUrlSource`, in the
+        # package-lock.json file. The output format is a newline separated list
+        # of entries, where each entry contains a JSON object path of the
+        # integrity field and the corresponding resolved file path, separated
+        # by a tab, ready for shell consumption
+        jqFindNullIntegrity = ''
+          # Processes dependencies entries as { key, value } pairs
+          def process(prefix):
+            (prefix + [ .key ]) as $path | .value |
+            (
+              # If we have an integrity attribute that is null, output an entry
+              if has("integrity") and .integrity == null
+              then
+                [ ($path + ["integrity"] | @json)
+                , (.resolved | ltrimstr("file://"))
+                ]
+              else empty
+              end
+            ,
+              # Recurse into .dependencies, this won't be necessary for
+              # lockfile version 2
+              if has("dependencies")
+              then .dependencies | to_entries[] | process($path + ["dependencies"])
+              else empty
+              end
+            );
+          .dependencies | to_entries[] | process(["dependencies"])
+          # Does the newline/tab separated thing, nice for shell consumption
+          | @tsv
+        '';
+
+        # A script for updating specific JSON paths (.path) with specific
+        # values (.value), as given in a list of objects, of an $original[0]
+        # JSON value
+        jqSetIntegrity = ''
+          reduce .[] as $update
+            ( $original[0]
+            ; . * setpath($update | .path; $update | .value)
+            )
+        '';
+
+        passAsFile = [ "jqFindNullIntegrity" "jqSetIntegrity" ];
+
         postPatch = ''
-          ln -sf ${patchedLockfile sourceOptions packageLockJson} package-lock.json
-          ln -sf ${patchedPackagefile packageJson} package.json
+          # Patches the lockfile at build time to replace the `"integrity":
+          # null` entries as set by `makeUrlSource` at eval time.
+          jq -r -f "$jqFindNullIntegrityPath" ${patchedLockfilePath} | while IFS=$'\t' read jsonpath file; do
+
+            # https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json#packages
+            # https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity#tools_for_generating_sri_hashes
+            hash="sha512-$(openssl dgst -sha512 -binary "$file" | openssl base64 -A)"
+
+            # Constructs a simple { path, value } JSON of the given arguments
+            jq -c --argjson path "$jsonpath" --arg value "$hash" -n '$ARGS.named'
+
+          done | jq -s --slurpfile original ${patchedLockfilePath} -f "$jqSetIntegrityPath" > package-lock.json
+
+          ln -sf ${patchedPackagefilePath} package.json
         '';
 
         buildPhase = ''
@@ -418,8 +533,8 @@ rec {
 
         passthru = passthru // {
           inherit nodejs;
-          lockfile = patchedLockfile packageLockJson;
-          packagesfile = patchedPackagefile packageJson;
+          lockfile = patchedLockfilePath;
+          packagesfile = patchedPackagefilePath;
         };
       } // cleanArgs);
 
diff --git a/tests/examples-projects/source-patching/package-lock.json b/tests/examples-projects/source-patching/package-lock.json
new file mode 100644
index 0000000..889f3e3
--- /dev/null
+++ b/tests/examples-projects/source-patching/package-lock.json
@@ -0,0 +1,13 @@
+{
+  "name": "source-patching",
+  "version": "1.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "custom-hello-world": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/custom-hello-world/-/custom-hello-world-1.0.3.tgz",
+      "integrity": "sha512-Rinkq1q+uLmgbDLZRNZrlyeK3G9e+o0gbVy6r948kySbau0ymwOPNybu4iIkb4R1gE5aZsQUHgtfCg2oZ4zfbw=="
+    }
+  }
+}
diff --git a/tests/examples-projects/source-patching/package.json b/tests/examples-projects/source-patching/package.json
new file mode 100644
index 0000000..d0ea4e4
--- /dev/null
+++ b/tests/examples-projects/source-patching/package.json
@@ -0,0 +1,11 @@
+{
+  "name": "source-patching",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "custom-hello-world": "^1.0.0"
+  }
+}
diff --git a/tests/examples-projects/source-patching/shell.nix b/tests/examples-projects/source-patching/shell.nix
new file mode 100644
index 0000000..e8073e5
--- /dev/null
+++ b/tests/examples-projects/source-patching/shell.nix
@@ -0,0 +1,25 @@
+{ npmlock2nix }:
+npmlock2nix.shell {
+  src = ./.;
+  node_modules_attrs = {
+    sourceMods = {
+      custom-hello-world = _: {
+        patches = builtins.toFile "custom-hello-world.patch" ''
+          diff --git a/lib/index.js b/lib/index.js
+          index 1f66513..64391a7 100644
+          --- a/lib/index.js
+          +++ b/lib/index.js
+          @@ -21,7 +21,7 @@ function generateHelloWorld({ comma, exclamation, lowercase }) {
+             if (comma)
+               helloWorldStr += ',';
+             
+          -  helloWorldStr += ' World';
+          +  helloWorldStr += ' Nix';
+           
+             if (exclamation)
+               helloWorldStr += '!';
+        '';
+      };
+    };
+  };
+}
diff --git a/tests/integration-tests/default.nix b/tests/integration-tests/default.nix
index 9ea5593..33e6aab 100644
--- a/tests/integration-tests/default.nix
+++ b/tests/integration-tests/default.nix
@@ -246,4 +246,12 @@ testLib.makeIntegrationTests {
       works
     '';
   };
+  source-patching = {
+    description = "Source patching works";
+    shell = callPackage ../examples-projects/source-patching/shell.nix { };
+    command = ''
+      node -e 'console.log(require("custom-hello-world")({}));'
+    '';
+    expected = "Hello Nix\n";
+  };
 }
diff --git a/tests/lib.nix b/tests/lib.nix
index cf2432e..8764b3c 100644
--- a/tests/lib.nix
+++ b/tests/lib.nix
@@ -10,7 +10,10 @@
   # Reads a given file (either drv, path or string) and returns it's sha256 hash
   hashFile = filename: builtins.hashString "sha256" (builtins.readFile filename);
 
-  noSourceOptions = { sourceHashFunc = _: null; };
+  noSourceOptions = {
+    sourceHashFunc = _: null;
+    nodejs = null;
+  };
 
   runTests = tests:
     let
diff --git a/tests/make-source.nix b/tests/make-source.nix
index ddf913d..dd44426 100644
--- a/tests/make-source.nix
+++ b/tests/make-source.nix
@@ -1,7 +1,10 @@
 { testLib, npmlock2nix }:
 let
   i = npmlock2nix.internal;
-  f = { sourceHashFunc = builtins.throw "Shouldn't be called"; };
+  f = {
+    sourceHashFunc = builtins.throw "Shouldn't be called";
+    nodejs = null;
+  };
 in
 testLib.runTests {
   testMakeSourceRegular = {