Add support for patching dependencies

Can be used to patchShebangs for specific packages, which with NPM
version 7.0 can't be done globally for all packages anymore

Could also be a replacement for preInstallLinks

API.md

@@ -16,6 +16,7 @@ The `node_modules` function takes an attribute set with the following attributes
 - **nodejs** *(default `nixpkgs.nodejs`, which is the Active LTS version)*: Node.js derivation to use
 - **preInstallLinks** *(default `{}`)*: Map of symlinks to create inside npm dependencies in the `node_modules` output (See [Concepts](#concepts) for details).
 - **githubSourceHashMap** *(default `{}`)*: Dependency hashes for evaluation in restricted mode (See [Concepts](#concepts) for details).
+- **sourceAttrs** *(default `{}`)*: Derivation attributes to apply to sources, allowing patching (See [Concepts](#concepts) for details)
 
 #### Notes
 - You may provide additional arguments accepted by `mkDerivation` all of which are going to be passed on.
@@ -114,3 +115,32 @@ npmlock2nix.build {
   };
 }
 ```
+
+### Source derivation attributes
+
+`node_modules` takes a `sourceAttrs` argument, which allows you to modify the source derivations of individual npm packages you depend on, mainly useful for adding Nix-specific fixes to packages. This could be used for patching interpreter or paths, or to replace vendored binaries with ones provided by Nix.
+
+The `sourceAttrs` argument expects an attribute set mapping npm package names to a function describing the modifications of that package. Each function receives an attribute set as an argument containing either a `version` attribute if the version is known, or a `github = { org, repo, rev, ref }` attribute if the package is fetched from GitHub. These values can be used to decide what the result of the function should be. The result of this function is passed to a `mkDerivation` call running mainly the [patch phase](https://nixos.org/manual/nixpkgs/stable/#ssec-patch-phase), meaning that [most `mkDerivation` arguments](https://nixos.org/manual/nixpkgs/stable/#chap-stdenv) can be returned from it. Of particular interest are `patches` and `postPatch`, in which `patchShebangs` can be called. Note that `patchShebangs` can only patch shebangs to binaries accessible in the derivation, which you can extend with `buildInputs`. For convenience, the correct version of `nodejs` is always included in `buildInputs`.
+
+```nix
+npmlock2nix.node_modules {
+  sourceMods = {
+    # sourceInfo either contains:
+    # - A version attribute
+    # - A github = { org, repo, rev, ref } attribute for GitHub sources
+    package-name = sourceInfo: {
+      buildInputs = [ somePackage ];
+      patches = [ somePatch ];
+      postPatch = ''
+        some script
+      '';
+      # ...
+    }
+
+    # Example
+    node-pre-gyp = sourceInfo: {
+      postPatch = "patchShebangs bin";
+    };
+  };
+}
+```

internal.nix

@@ -1,4 +1,4 @@
-{ nodejs, stdenv, mkShell, lib, fetchurl, writeText, writeTextFile, runCommand, fetchFromGitHub }:
+{ nodejs, jq, openssl, stdenv, mkShell, lib, fetchurl, writeText, writeTextFile, runCommand, fetchFromGitHub }:
 rec {
   default_nodejs = nodejs;
 
@@ -60,7 +60,7 @@ rec {
   # hash attribute it will provide the value to `fetchFromGitHub` which will
   # also work in restricted evaluation.
   # Type: Set -> Path
-  buildTgzFromGitHub = { name, org, repo, rev, ref, hash ? null }:
+  buildTgzFromGitHub = { name, org, repo, rev, ref, hash ? null, sourceOptions ? { } }:
     let
       src =
         if hash != null then
@@ -76,20 +76,44 @@ rec {
             inherit rev ref;
             allRefs = true;
           };
+
+      attrs = {
+        pname = name;
+        version = ref;
+        src = src;
+      } // lib.optionalAttrs (sourceOptions ? sourceAttrs.${name})
+        (sourceOptions.sourceAttrs.${name} {
+          github = { inherit org repo rev ref; };
+        });
     in
-    runCommand
-      name
-      { } ''
-      set +x
-      tar -C ${src} -czf $out ./
+    packTgz sourceOptions.nodejs attrs;
+
+  # Description: Packs a source directory into a .tgz tar archive, allowing the
+  # source to be modified using derivation attributes. If the source is an
+  # archive, it gets unpacked first.
+  # Type: Path -> Set -> Path
+  packTgz = nodejs: attrs@{ pname, version, src, ... }: stdenv.mkDerivation ({
+    name = "${pname}-${version}.tgz";
+    phases = "unpackPhase patchPhase installPhase";
+    installPhase = ''
+      runHook preInstall
+      tar -C . -czf $out ./
+      runHook postInstall
     '';
+  } // attrs // {
+    buildInputs = attrs.buildInputs or [ ] ++ [
+      # Allows patchShebangs in postPatch to patch shebangs to nodejs
+      nodejs
+    ];
+  });
 
   # Description: Turns a dependency with a from field of the format
   # `github:org/repo#revision` into a git fetcher. The fetcher can
   # receive a hash value by calling 'sourceHashFunc' if a source hash
-  # map has been provided. Otherwise the function yields `null`.
+  # map has been provided. Otherwise the function yields `null`. Patches
+  # specified with sourceAttrs will be applied
   # Type: { sourceHashFunc :: Fn } -> String -> Set -> Path
-  makeGithubSource = { sourceHashFunc, ... }: name: dependency:
+  makeGithubSource = sourceOptions@{ sourceHashFunc, ... }: name: dependency:
     assert !(dependency ? version) ->
       builtins.throw "version` attribute missing from `${name}`";
     assert (lib.hasPrefix "github: " dependency.version) -> builtins.throw "invalid prefix for `version` field of `${name}` expected `github:`, got: `${dependency.version}`.";
@@ -106,6 +130,7 @@ rec {
         ref = f.rev;
         inherit (v) org repo rev;
         hash = sourceHashFunc { type = "github"; value = v; };
+        inherit sourceOptions;
       };
     in
     (builtins.removeAttrs dependency [ "from" ]) // {
@@ -125,6 +150,35 @@ rec {
   shouldUseVersionAsUrl = dependency:
     dependency ? version && dependency ? integrity && ! (dependency ? resolved) && looksLikeUrl dependency.version;
 
+  # Description: Replaces the `resolved` field of a dependency with a
+  # prefetched version from the Nix store. Patches specified with sourceAttrs
+  # will be applied, in which case the `integrity` attribute is set to `null`,
+  # in order to be recomputer later
+  # Type: { sourceAttrs :: Fn, nodejs :: Package } -> String -> Set -> Set
+  makeUrlSource = { sourceAttrs ? { }, nodejs, ... }: name: dependency:
+    let
+      src = fetchurl (makeSourceAttrs name dependency);
+      attrs = {
+        pname = name;
+        version = dependency.version;
+        src = src;
+      } // sourceAttrs.${name} {
+        inherit (dependency) version;
+      };
+      tgz =
+        if sourceAttrs ? ${name}
+        # If we have modification to this source, unpack the tgz, apply the
+        # patches and repack the tgz
+        then packTgz nodejs attrs
+        else src;
+      resolved = "file://" + toString tgz;
+    in
+    dependency // { inherit resolved; } // lib.optionalAttrs (sourceAttrs ? ${name}) {
+      # Integrity was tampered with due to the source attributes, so it needs
+      # to be recalculated, which is done in the node_modules builder
+      integrity = null;
+    };
+
   # Description: Turns an npm lockfile dependency into a fetchurl derivation
   # Type: { sourceHashFunc :: Fn } -> String -> Set -> Derivation
   makeSource = sourceOptions: name: dependency:
@@ -133,7 +187,7 @@ rec {
     assert (builtins.typeOf dependency != "set") ->
       throw "Specification of dependency ${toString name} must be a set";
     if dependency ? resolved && dependency ? integrity then
-      dependency // { resolved = "file://" + (toString (fetchurl (makeSourceAttrs name dependency))); }
+      makeUrlSource sourceOptions name dependency
     else if dependency ? from && dependency ? version then
       makeGithubSource sourceOptions name dependency
     else if shouldUseVersionAsUrl dependency then
@@ -156,7 +210,7 @@ rec {
 
   # Description: Turns a github string reference into a store path with a tgz of the reference
   # Type: Fn -> String -> String -> Path
-  stringToTgzPath = { sourceHashFunc, ... }: name: str:
+  stringToTgzPath = sourceOptions@{ sourceHashFunc, ... }: name: str:
     let
       gitAttrs = parseGitHubRef str;
     in
@@ -165,6 +219,7 @@ rec {
       ref = gitAttrs.rev;
       inherit (gitAttrs) org repo rev;
       hash = sourceHashFunc { type = "github"; value = gitAttrs; };
+      inherit sourceOptions;
     };
 
   # Description: Patch the `requires` attributes of a dependency spec to refer to paths in the store
@@ -314,20 +369,25 @@ rec {
     , preBuild ? ""
     , postBuild ? ""
     , preInstallLinks ? { } # set that describes which files should be linked in a specific packages folder
+    , sourceAttrs ? { }
     , githubSourceHashMap ? { }
     , passthru ? { }
     , ...
     }@args:
       assert (builtins.typeOf preInstallLinks != "set") ->
         throw "`preInstallLinks` must be an attributeset of attributesets";
       let
-        cleanArgs = builtins.removeAttrs args [ "src" "packageJson" "packageLockJson" "buildInputs" "nativeBuildInputs" "nodejs" "preBuild" "postBuild" "preInstallLinks" "githubSourceHashMap" ];
+        cleanArgs = builtins.removeAttrs args [ "src" "packageJson" "packageLockJson" "buildInputs" "nativeBuildInputs" "nodejs" "preBuild" "postBuild" "preInstallLinks" "sourceAttrs" "githubSourceHashMap" ];
         lockfile = readLockfile packageLockJson;
 
         sourceOptions = {
           sourceHashFunc = sourceHashFunc githubSourceHashMap;
+          inherit nodejs sourceAttrs;
         };
 
+        patchedLockfilePath = patchedLockfile sourceOptions packageLockJson;
+        patchedPackagefilePath = patchedPackagefile packageJson;
+
         preinstall_node_modules = writeTextFile {
           name = "prepare";
           destination = "/node_modules/.hooks/prepare";
@@ -372,6 +432,8 @@ rec {
         dontUnpack = true;
 
         nativeBuildInputs = nativeBuildInputs ++ [
+          jq
+          openssl
           nodejs
         ];
 
@@ -387,9 +449,65 @@ rec {
           export HOME=$(mktemp -d)
         '';
 
+        # A jq filter for finding dependencies with an integrity field of
+        # `null`, as set at evaluation time by `makeUrlSource`, in the
+        # package-lock.json file. The output format is a newline separated list
+        # of entries, where each entry contains a JSON object path of the
+        # integrity field and the corresponding resolved file path, separated
+        # by a tab, ready for shell consumption
+        jqFindNullIntegrity = ''
+          # Processes dependencies entries as { key, value } pairs
+          def process(prefix):
+            (prefix + [ .key ]) as $path | .value |
+            (
+              # If we have an integrity attribute that is null, output an entry
+              if has("integrity") and .integrity == null
+              then
+                [ ($path + ["integrity"] | @json)
+                , (.resolved | ltrimstr("file://"))
+                ]
+              else empty
+              end
+            ,
+              # Recurse into .dependencies, this won't be necessary for
+              # lockfile version 2
+              if has("dependencies")
+              then .dependencies | to_entries[] | process($path + ["dependencies"])
+              else empty
+              end
+            );
+          .dependencies | to_entries[] | process(["dependencies"])
+          # Does the newline/tab separated thing, nice for shell consumption
+          | @tsv
+        '';
+
+        # A script for updating specific JSON paths (.path) with specific
+        # values (.value), as given in a list of objects, of an $original[0]
+        # JSON value
+        jqSetIntegrity = ''
+          reduce .[] as $update
+            ( $original[0]
+            ; . * setpath($update | .path; $update | .value)
+            )
+        '';
+
+        passAsFile = [ "jqFindNullIntegrity" "jqSetIntegrity" ];
+
         postPatch = ''
-          ln -sf ${patchedLockfile sourceOptions packageLockJson} package-lock.json
-          ln -sf ${patchedPackagefile packageJson} package.json
+          # Patches the lockfile at build time to replace the `"integrity":
+          # null` entries as set by `makeUrlSource` at eval time.
+          jq -r -f "$jqFindNullIntegrityPath" ${patchedLockfilePath} | while IFS=$'\t' read jsonpath file; do
+
+            # https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json#packages
+            # https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity#tools_for_generating_sri_hashes
+            hash="sha512-$(openssl dgst -sha512 -binary "$file" | openssl base64 -A)"
+
+            # Constructs a simple { path, value } JSON of the given arguments
+            jq -c --argjson path "$jsonpath" --arg value "$hash" -n '$ARGS.named'
+
+          done | jq -s --slurpfile original ${patchedLockfilePath} -f "$jqSetIntegrityPath" > package-lock.json
+
+          ln -sf ${patchedPackagefilePath} package.json
         '';
 
         buildPhase = ''
@@ -418,8 +536,8 @@ rec {
 
         passthru = passthru // {
           inherit nodejs;
-          lockfile = patchedLockfile packageLockJson;
-          packagesfile = patchedPackagefile packageJson;
+          lockfile = patchedLockfilePath;
+          packagesfile = patchedPackagefilePath;
         };
       } // cleanArgs);
 

tests/examples-projects/source-patching/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "source-patching",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "custom-hello-world": "^1.0.0"
+  }
+}

tests/examples-projects/source-patching/shell.nix

@@ -0,0 +1,25 @@
+{ npmlock2nix }:
+npmlock2nix.shell {
+  src = ./.;
+  node_modules_attrs = {
+    sourceAttrs = {
+      custom-hello-world = _: {
+        patches = builtins.toFile "custom-hello-world.patch" ''
+          diff --git a/lib/index.js b/lib/index.js
+          index 1f66513..64391a7 100644
+          --- a/lib/index.js
+          +++ b/lib/index.js
+          @@ -21,7 +21,7 @@ function generateHelloWorld({ comma, exclamation, lowercase }) {
+             if (comma)
+               helloWorldStr += ',';
+             
+          -  helloWorldStr += ' World';
+          +  helloWorldStr += ' Nix';
+           
+             if (exclamation)
+               helloWorldStr += '!';
+        '';
+      };
+    };
+  };
+}

tests/integration-tests/default.nix

@@ -246,4 +246,12 @@ testLib.makeIntegrationTests {
       works
     '';
   };
+  source-patching = {
+    description = "Source patching works";
+    shell = callPackage ../examples-projects/source-patching/shell.nix { };
+    command = ''
+      node -e 'console.log(require("custom-hello-world")({}));'
+    '';
+    expected = "Hello Nix\n";
+  };
 }

tests/lib.nix

@@ -10,7 +10,10 @@
   # Reads a given file (either drv, path or string) and returns it's sha256 hash
   hashFile = filename: builtins.hashString "sha256" (builtins.readFile filename);
 
-  noSourceOptions = { sourceHashFunc = _: null; };
+  noSourceOptions = {
+    sourceHashFunc = _: null;
+    nodejs = null;
+  };
 
   runTests = tests:
     let

tests/make-source.nix

@@ -1,7 +1,10 @@
 { testLib, npmlock2nix }:
 let
   i = npmlock2nix.internal;
-  f = { sourceHashFunc = builtins.throw "Shouldn't be called"; };
+  f = {
+    sourceHashFunc = builtins.throw "Shouldn't be called";
+    nodejs = null;
+  };
 in
 testLib.runTests {
   testMakeSourceRegular = {