Merge pull request #151 from infinisil/source-options

Allow patching dependency sources

API.md

@@ -16,6 +16,7 @@ The `node_modules` function takes an attribute set with the following attributes
 - **nodejs** *(default `nixpkgs.nodejs`, which is the Active LTS version)*: Node.js derivation to use
 - **preInstallLinks** *(default `{}`)*: Map of symlinks to create inside npm dependencies in the `node_modules` output (See [Concepts](#concepts) for details).
 - **githubSourceHashMap** *(default `{}`)*: Dependency hashes for evaluation in restricted mode (See [Concepts](#concepts) for details).
+- **sourceOverrides** *(default `{}`)*: Derivation attributes to apply to sources, allowing patching (See the [source derivation overrides](#source-derivation-overrides) concept for details)
 
 #### Notes
 - You may provide additional arguments accepted by `mkDerivation` all of which are going to be passed on.
@@ -114,3 +115,32 @@ npmlock2nix.build {
   };
 }
 ```
+
+### Source derivation overrides
+
+`node_modules` takes a `sourceOverrides` argument, which allows you to modify the source derivations of individual npm packages you depend on, mainly useful for adding Nix-specific fixes to packages. This could be used for patching interpreter or paths, or to replace vendored binaries with ones provided by Nix.
+
+The `sourceOverrides` argument expects an attribute set mapping npm package names to a function describing the modifications of that package. Each function receives an attribute set as a first argument, containing either a `version` attribute if the version is known, or a `github = { org, repo, rev, ref }` attribute if the package is fetched from GitHub. These values can be used to have different overrides depending on the version. The function receives another argument which is the derivation of the fetched source, which can be modified using `.overrideAttrs`. The fetched source mainly runs the [patch phase](https://nixos.org/manual/nixpkgs/stable/#ssec-patch-phase), so of particular interest are the `patches` and `postPatch` attributes, in which `patchShebangs` can be called. Note that `patchShebangs` can only patch shebangs to binaries accessible in the derivation, which you can extend with `buildInputs`. For convenience, the correct version of `nodejs` is always included in `buildInputs`.
+
+```nix
+npmlock2nix.node_modules {
+  sourceOverrides = {
+    # sourceInfo either contains:
+    # - A version attribute
+    # - A github = { org, repo, rev, ref } attribute for GitHub sources
+    package-name = sourceInfo: drv: drv.overrideAttrs (old: {
+      buildInputs = [ somePackage ];
+      patches = [ somePatch ];
+      postPatch = ''
+        some script
+      '';
+      # ...
+    })
+
+    # Example
+    node-pre-gyp = sourceInfo: drv: drv.overrideAttrs (old: {
+      postPatch = "patchShebangs bin";
+    });
+  };
+}
+```

tests/examples-projects/source-patching/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "source-patching",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "custom-hello-world": "^1.0.0"
+  }
+}

tests/examples-projects/source-patching/shell.nix

@@ -0,0 +1,25 @@
+{ npmlock2nix }:
+npmlock2nix.shell {
+  src = ./.;
+  node_modules_attrs = {
+    sourceOverrides = {
+      custom-hello-world = sourceInfo: drv: drv.overrideAttrs (old: {
+        patches = builtins.toFile "custom-hello-world.patch" ''
+          diff --git a/lib/index.js b/lib/index.js
+          index 1f66513..64391a7 100644
+          --- a/lib/index.js
+          +++ b/lib/index.js
+          @@ -21,7 +21,7 @@ function generateHelloWorld({ comma, exclamation, lowercase }) {
+             if (comma)
+               helloWorldStr += ',';
+             
+          -  helloWorldStr += ' World';
+          +  helloWorldStr += ' Nix';
+           
+             if (exclamation)
+               helloWorldStr += '!';
+        '';
+      });
+    };
+  };
+}

tests/integration-tests/default.nix

@@ -284,4 +284,13 @@ testLib.makeIntegrationTests {
       '';
     };
   };
+
+  source-patching = {
+    description = "Source patching works";
+    shell = callPackage ../examples-projects/source-patching/shell.nix { };
+    command = ''
+      node -e 'console.log(require("custom-hello-world")({}));'
+    '';
+    expected = "Hello Nix\n";
+  };
 }

tests/lib.nix

@@ -10,7 +10,10 @@
   # Reads a given file (either drv, path or string) and returns it's sha256 hash
   hashFile = filename: builtins.hashString "sha256" (builtins.readFile filename);
 
-  noGithubHashes = (_: null);
+  noSourceOptions = {
+    sourceHashFunc = _: null;
+    nodejs = null;
+  };
 
   runTests = tests:
     let

tests/make-github-source.nix

@@ -1,6 +1,6 @@
 { lib, npmlock2nix, testLib }:
 let
-  inherit (testLib) noGithubHashes;
+  inherit (testLib) noSourceOptions;
   i = npmlock2nix.internal;
 
   testDependency = {
@@ -12,7 +12,7 @@ in
   testSimpleCase = {
     expr =
       let
-        version = (i.makeGithubSource noGithubHashes "leftpad" testDependency).version;
+        version = (i.makeGithubSource noSourceOptions "leftpad" testDependency).version;
       in
       lib.hasPrefix "file:///nix/store" version;
     expected = true;
@@ -21,7 +21,7 @@ in
   testDropsFrom = {
     expr =
       let
-        dep = i.makeGithubSource noGithubHashes "leftpad" testDependency;
+        dep = i.makeGithubSource noSourceOptions "leftpad" testDependency;
       in
       dep ? from;
     expected = false;

tests/make-source.nix

@@ -1,7 +1,10 @@
 { testLib, npmlock2nix }:
 let
   i = npmlock2nix.internal;
-  f = builtins.throw "Shouldn't be called";
+  f = {
+    sourceHashFunc = builtins.throw "Shouldn't be called";
+    nodejs = null;
+  };
 in
 testLib.runTests {
   testMakeSourceRegular = {

tests/patch-lockfile.nix

@@ -1,33 +1,33 @@
 { npmlock2nix, testLib, lib }:
 let
-  inherit (testLib) noGithubHashes;
+  inherit (testLib) noSourceOptions;
 in
 testLib.runTests {
 
   testPatchDependencyHandlesGitHubRefsInRequires = {
     expr =
       let
-        libxmljsUrl = (npmlock2nix.internal.patchDependency noGithubHashes "test" {
+        libxmljsUrl = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" {
           version = "github:tmcw/leftpad#db1442a0556c2b133627ffebf455a78a1ced64b9";
           from = "github:tmcw/leftpad#db1442a0556c2b133627ffebf455a78a1ced64b9";
           integrity = "sha512-8/UvHFG90J4O4QNRzb0jB5Ni1QuvuB7XFTLfDMQnCzAsFemF29VKnNGUESFFcSP/r5WWh/PMe0YRz90+3IqsUA==";
           requires = {
             libxmljs = "github:znerol/libxmljs#0517e063347ea2532c9fdf38dc47878c628bf0ae";
           };
         }
-        ).requires.libxmljs;
+        ).result.requires.libxmljs;
       in
       lib.hasPrefix builtins.storeDir libxmljsUrl;
     expected = true;
   };
 
   testBundledDependenciesAreRetained = {
-    expr = npmlock2nix.internal.patchDependency noGithubHashes "test" {
+    expr = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" {
       bundled = true;
       integrity = "sha1-hrGk3k+s4YCsVFqD8VA1I9j+0RU=";
       something = "bar";
       dependencies = { };
-    };
+    }).result;
     expected = {
       bundled = true;
       integrity = "sha1-hrGk3k+s4YCsVFqD8VA1I9j+0RU=";
@@ -37,18 +37,18 @@ testLib.runTests {
   };
 
   testPatchLockfileWithoutDependencies = {
-    expr = (npmlock2nix.internal.patchLockfile noGithubHashes ./examples-projects/no-dependencies/package-lock.json).dependencies;
+    expr = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/no-dependencies/package-lock.json).result.dependencies;
     expected = { };
   };
 
   testPatchDependencyDoesntDropAttributes = {
-    expr = npmlock2nix.internal.patchDependency noGithubHashes "test" {
+    expr = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" {
       a = 1;
       foo = "something";
       resolved = "https://examples.com/something.tgz";
       integrity = "sha1-00000000000000000000000+0RU=";
       dependencies = { };
-    };
+    }).result;
     expected = {
       a = 1;
       foo = "something";
@@ -59,7 +59,7 @@ testLib.runTests {
   };
 
   testPatchDependencyPatchesDependenciesRecursively = {
-    expr = npmlock2nix.internal.patchDependency noGithubHashes "test" {
+    expr = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" {
       a = 1;
       foo = "something";
       resolved = "https://examples.com/something.tgz";
@@ -68,7 +68,7 @@ testLib.runTests {
         resolved = "https://examples.com/somethingelse.tgz";
         integrity = "sha1-00000000000000000000000+00U=";
       };
-    };
+    }).result;
 
     expected = {
       a = 1;
@@ -85,7 +85,7 @@ testLib.runTests {
   testPatchLockfileTurnsUrlsIntoStorePaths = {
     expr =
       let
-        deps = (npmlock2nix.internal.patchLockfile noGithubHashes ./examples-projects/single-dependency/package-lock.json).dependencies;
+        deps = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/single-dependency/package-lock.json).result.dependencies;
       in
       lib.count (dep: lib.hasPrefix "file:///nix/store/" dep.resolved) (lib.attrValues deps);
     expected = 1;
@@ -94,19 +94,19 @@ testLib.runTests {
   testPatchLockfileTurnsGitHubUrlsIntoStorePaths = {
     expr =
       let
-        leftpad = (npmlock2nix.internal.patchLockfile noGithubHashes ./examples-projects/github-dependency/package-lock.json).dependencies.leftpad;
+        leftpad = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/github-dependency/package-lock.json).result.dependencies.leftpad;
       in
       lib.hasPrefix ("file://" + builtins.storeDir) leftpad.version;
     expected = true;
   };
 
   testConvertPatchedLockfileToJSON = {
-    expr = builtins.typeOf (builtins.toJSON (npmlock2nix.internal.patchLockfile noGithubHashes ./examples-projects/nested-dependencies/package-lock.json)) == "string";
+    expr = builtins.typeOf (builtins.toJSON (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/nested-dependencies/package-lock.json).result) == "string";
     expected = true;
   };
 
   testPatchedLockFile = {
-    expr = testLib.hashFile (npmlock2nix.internal.patchedLockfile noGithubHashes ./examples-projects/nested-dependencies/package-lock.json);
+    expr = testLib.hashFile (npmlock2nix.internal.patchedLockfile noSourceOptions ./examples-projects/nested-dependencies/package-lock.json).result;
     expected = "980323c3a53d86ab6886f21882936cfe7c06ac633993f16431d79e3185084414";
   };
 

tests/patch-packagefile.nix

@@ -1,19 +1,18 @@
 { lib, npmlock2nix, testLib }:
 let
-  inherit (testLib) noGithubHashes;
   i = npmlock2nix.internal;
 in
 (testLib.runTests {
   testTurnsGitHubRefsToWildcards = {
-    expr = (npmlock2nix.internal.patchPackagefile noGithubHashes ./examples-projects/github-dependency/package.json).dependencies.leftpad;
+    expr = (npmlock2nix.internal.patchPackagefile ./examples-projects/github-dependency/package.json).dependencies.leftpad;
     expected = "*";
   };
   testHandlesBranches = {
-    expr = (npmlock2nix.internal.patchPackagefile noGithubHashes ./examples-projects/github-dependency-branch/package.json).dependencies.leftpad;
+    expr = (npmlock2nix.internal.patchPackagefile ./examples-projects/github-dependency-branch/package.json).dependencies.leftpad;
     expected = "*";
   };
   testHandlesDevDependencies = {
-    expr = (npmlock2nix.internal.patchPackagefile noGithubHashes ./examples-projects/github-dev-dependency/package.json).devDependencies.leftpad;
+    expr = (npmlock2nix.internal.patchPackagefile ./examples-projects/github-dev-dependency/package.json).devDependencies.leftpad;
     expected = "*";
   };
 })