From bbb6fd47a820a550568277dc073d487a5b5f73b6 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 31 Mar 2022 18:50:08 +0200 Subject: [PATCH] Collect necessary integrity changes with Nix Instead of jq --- internal.nix | 113 +++++++++++++++++++-------------------- tests/patch-lockfile.nix | 26 ++++----- 2 files changed, 68 insertions(+), 71 deletions(-) diff --git a/internal.nix b/internal.nix index b91111c..ae5ef67 100644 --- a/internal.nix +++ b/internal.nix @@ -234,8 +234,8 @@ rec { # Description: Patches a single lockfile dependency (recursively) by replacing the resolved URL with a store path - # Type: { sourceHashFunc :: Fn } -> String -> Set -> Set - patchDependency = sourceOptions: name: spec: + # Type: List String -> { sourceHashFunc :: Fn } -> String -> Set -> { result :: Set, integrityUpdates :: List { path, file } } + patchDependency = path: sourceOptions: name: spec: assert (builtins.typeOf name != "string") -> throw "Name of dependency ${toString name} must be a string"; assert (builtins.typeOf spec != "set") -> @@ -245,22 +245,38 @@ rec { hasGitHubRequires = spec: (spec ? requires) && (lib.any (x: lib.hasPrefix "github:" x) (lib.attrValues spec.requires)); patchSource = lib.optionalAttrs (!isBundled) (makeSource sourceOptions name spec); patchRequiresSources = lib.optionalAttrs (hasGitHubRequires spec) { requires = (patchRequires sourceOptions name spec.requires); }; - patchDependenciesSources = lib.optionalAttrs (spec ? dependencies) { dependencies = lib.mapAttrs (patchDependency sourceOptions) spec.dependencies; }; - in - # For our purposes we need a dependency with + nestedDependencies = lib.mapAttrs (name: patchDependency (path ++ [ name ]) sourceOptions name) spec.dependencies; + patchDependenciesSources = lib.optionalAttrs (spec ? dependencies) { dependencies = lib.mapAttrs (_: value: value.result) nestedDependencies; }; + nestedIntegrityUpdates = lib.concatMap (value: value.integrityUpdates) (lib.attrValues nestedDependencies); + + # For our purposes we need a dependency with # - `resolved` set to a path in the nix store (`patchSource`) # - All `requires` entries of this dependency that are set to github URLs set to a path in the nix store (`patchRequiresSources`) # - This needs to be done recursively for all `dependencies` in the lockfile (`patchDependenciesSources`) - (spec // patchSource // patchRequiresSources // patchDependenciesSources); + result = spec // patchSource // patchRequiresSources // patchDependenciesSources; + in + { + result = result; + integrityUpdates = lib.optional (result ? resolved && result ? integrity && result.integrity == null) { + inherit path; + file = lib.removePrefix "file://" result.resolved; + }; + }; # Description: Takes a Path to a lockfile and returns the patched version as attribute set - # Type: { sourceHashFunc :: Fn } -> Path -> Set + # Type: { sourceHashFunc :: Fn } -> Path -> { result :: Set, integrityUpdates :: List { path, file } } patchLockfile = sourceOptions: file: assert (builtins.typeOf file != "path" && builtins.typeOf file != "string") -> throw "file ${toString file} must be a path or string"; - let content = readLockfile file; in - content // { - dependencies = lib.mapAttrs (patchDependency sourceOptions) content.dependencies; + let + content = readLockfile file; + dependencies = lib.mapAttrs (name: patchDependency [ name ] sourceOptions name) content.dependencies; + in + { + result = content // { + dependencies = lib.mapAttrs (_: value: value.result) dependencies; + }; + integrityUpdates = lib.concatMap (value: value.integrityUpdates) (lib.attrValues dependencies); }; # Description: Rewrite all the `github:` references to wildcards. @@ -295,9 +311,15 @@ rec { ); # Description: Takes a Path to a lockfile and returns the patched version as file in the Nix store - # Type: { sourceHashFunc :: Fn } -> Path -> Derivation - patchedLockfile = sourceOptions: file: writeText "package-lock.json" - (builtins.toJSON (patchLockfile sourceOptions file)); + # Type: { sourceHashFunc :: Fn } -> Path -> { result :: Derivation, integrityUpdates :: List { path, file } } + patchedLockfile = sourceOptions: file: + let + patched = patchLockfile sourceOptions file; + in + { + result = writeText "package-lock.json" (builtins.toJSON patched.result); + integrityUpdates = patched.integrityUpdates; + }; # Description: Turn a derivation (with name & src attribute) into a directory containing the unpacked sources # Type: Derivation -> Derivation @@ -389,7 +411,7 @@ rec { inherit nodejs sourceAttrs; }; - patchedLockfilePath = patchedLockfile sourceOptions packageLockJson; + patchedLockfile' = patchedLockfile sourceOptions packageLockJson; patchedPackagefilePath = patchedPackagefile packageJson; preinstall_node_modules = writeTextFile { @@ -437,6 +459,7 @@ rec { nativeBuildInputs = nativeBuildInputs ++ [ jq + ] ++ lib.optionals (patchedLockfile'.integrityUpdates != [ ]) [ openssl nodejs ]; @@ -453,38 +476,6 @@ rec { export HOME=$(mktemp -d) ''; - # A jq filter for finding dependencies with an integrity field of - # `null`, as set at evaluation time by `makeUrlSource`, in the - # package-lock.json file. The output format is a newline separated list - # of entries, where each entry contains a JSON object path of the - # integrity field and the corresponding resolved file path, separated - # by a tab, ready for shell consumption - jqFindNullIntegrity = '' - # Processes dependencies entries as { key, value } pairs - def process(prefix): - (prefix + [ .key ]) as $path | .value | - ( - # If we have an integrity attribute that is null, output an entry - if has("integrity") and .integrity == null - then - [ ($path + ["integrity"] | @json) - , (.resolved | ltrimstr("file://")) - ] - else empty - end - , - # Recurse into .dependencies, this won't be necessary for - # lockfile version 2 - if has("dependencies") - then .dependencies | to_entries[] | process($path + ["dependencies"]) - else empty - end - ); - .dependencies | to_entries[] | process(["dependencies"]) - # Does the newline/tab separated thing, nice for shell consumption - | @tsv - ''; - # A script for updating specific JSON paths (.path) with specific # values (.value), as given in a list of objects, of an $original[0] # JSON value @@ -495,21 +486,27 @@ rec { ) ''; - passAsFile = [ "jqFindNullIntegrity" "jqSetIntegrity" ]; + passAsFile = [ "jqSetIntegrity" ]; postPatch = '' # Patches the lockfile at build time to replace the `"integrity": # null` entries as set by `makeUrlSource` at eval time. - jq -r -f "$jqFindNullIntegrityPath" ${patchedLockfilePath} | while IFS=$'\t' read jsonpath file; do - - # https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json#packages - # https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity#tools_for_generating_sri_hashes - hash="sha512-$(openssl dgst -sha512 -binary "$file" | openssl base64 -A)" - - # Constructs a simple { path, value } JSON of the given arguments - jq -c --argjson path "$jsonpath" --arg value "$hash" -n '$ARGS.named' - - done | jq -s --slurpfile original ${patchedLockfilePath} -f "$jqSetIntegrityPath" > package-lock.json + # integrityUpdates is a list of { file, path } + ${if patchedLockfile'.integrityUpdates == [] then '' + cp ${patchedLockfile'.result} package-lock.json + '' else '' + { + ${lib.concatMapStrings ({ file, path }: '' + # https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json#packages + # https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity#tools_for_generating_sri_hashes + hash="sha512-$(openssl dgst -sha512 -binary ${lib.escapeShellArg file} | openssl base64 -A)" + + # Constructs a simple { path, value } JSON of the given arguments + jq -c --argjson path ${lib.escapeShellArg (builtins.toJSON path)} --arg value "$hash" -n '$ARGS.named' + '') patchedLockfile'.integrityUpdates} + } | jq -s --slurpfile original ${patchedLockfile'.result} -f "$jqSetIntegrityPath" > package-lock.json + set +x + ''} ln -sf ${patchedPackagefilePath} package.json ''; @@ -540,7 +537,7 @@ rec { passthru = passthru // { inherit nodejs; - lockfile = patchedLockfilePath; + lockfile = patchedLockfile'.result; packagesfile = patchedPackagefilePath; }; } // cleanArgs); diff --git a/tests/patch-lockfile.nix b/tests/patch-lockfile.nix index 20a0dec..2db2051 100644 --- a/tests/patch-lockfile.nix +++ b/tests/patch-lockfile.nix @@ -7,7 +7,7 @@ testLib.runTests { testPatchDependencyHandlesGitHubRefsInRequires = { expr = let - libxmljsUrl = (npmlock2nix.internal.patchDependency noSourceOptions "test" { + libxmljsUrl = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" { version = "github:tmcw/leftpad#db1442a0556c2b133627ffebf455a78a1ced64b9"; from = "github:tmcw/leftpad#db1442a0556c2b133627ffebf455a78a1ced64b9"; integrity = "sha512-8/UvHFG90J4O4QNRzb0jB5Ni1QuvuB7XFTLfDMQnCzAsFemF29VKnNGUESFFcSP/r5WWh/PMe0YRz90+3IqsUA=="; @@ -15,19 +15,19 @@ testLib.runTests { libxmljs = "github:znerol/libxmljs#0517e063347ea2532c9fdf38dc47878c628bf0ae"; }; } - ).requires.libxmljs; + ).result.requires.libxmljs; in lib.hasPrefix builtins.storeDir libxmljsUrl; expected = true; }; testBundledDependenciesAreRetained = { - expr = npmlock2nix.internal.patchDependency noSourceOptions "test" { + expr = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" { bundled = true; integrity = "sha1-hrGk3k+s4YCsVFqD8VA1I9j+0RU="; something = "bar"; dependencies = { }; - }; + }).result; expected = { bundled = true; integrity = "sha1-hrGk3k+s4YCsVFqD8VA1I9j+0RU="; @@ -37,18 +37,18 @@ testLib.runTests { }; testPatchLockfileWithoutDependencies = { - expr = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/no-dependencies/package-lock.json).dependencies; + expr = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/no-dependencies/package-lock.json).result.dependencies; expected = { }; }; testPatchDependencyDoesntDropAttributes = { - expr = npmlock2nix.internal.patchDependency noSourceOptions "test" { + expr = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" { a = 1; foo = "something"; resolved = "https://examples.com/something.tgz"; integrity = "sha1-00000000000000000000000+0RU="; dependencies = { }; - }; + }).result; expected = { a = 1; foo = "something"; @@ -59,7 +59,7 @@ testLib.runTests { }; testPatchDependencyPatchesDependenciesRecursively = { - expr = npmlock2nix.internal.patchDependency noSourceOptions "test" { + expr = (npmlock2nix.internal.patchDependency [ ] noSourceOptions "test" { a = 1; foo = "something"; resolved = "https://examples.com/something.tgz"; @@ -68,7 +68,7 @@ testLib.runTests { resolved = "https://examples.com/somethingelse.tgz"; integrity = "sha1-00000000000000000000000+00U="; }; - }; + }).result; expected = { a = 1; @@ -85,7 +85,7 @@ testLib.runTests { testPatchLockfileTurnsUrlsIntoStorePaths = { expr = let - deps = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/single-dependency/package-lock.json).dependencies; + deps = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/single-dependency/package-lock.json).result.dependencies; in lib.count (dep: lib.hasPrefix "file:///nix/store/" dep.resolved) (lib.attrValues deps); expected = 1; @@ -94,19 +94,19 @@ testLib.runTests { testPatchLockfileTurnsGitHubUrlsIntoStorePaths = { expr = let - leftpad = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/github-dependency/package-lock.json).dependencies.leftpad; + leftpad = (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/github-dependency/package-lock.json).result.dependencies.leftpad; in lib.hasPrefix ("file://" + builtins.storeDir) leftpad.version; expected = true; }; testConvertPatchedLockfileToJSON = { - expr = builtins.typeOf (builtins.toJSON (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/nested-dependencies/package-lock.json)) == "string"; + expr = builtins.typeOf (builtins.toJSON (npmlock2nix.internal.patchLockfile noSourceOptions ./examples-projects/nested-dependencies/package-lock.json).result) == "string"; expected = true; }; testPatchedLockFile = { - expr = testLib.hashFile (npmlock2nix.internal.patchedLockfile noSourceOptions ./examples-projects/nested-dependencies/package-lock.json); + expr = testLib.hashFile (npmlock2nix.internal.patchedLockfile noSourceOptions ./examples-projects/nested-dependencies/package-lock.json).result; expected = "980323c3a53d86ab6886f21882936cfe7c06ac633993f16431d79e3185084414"; };