WIP: Distribute prebuilt binaries via the npm registry

- Remove all custom download logic for prebuilt binaries - Add scripts to populate package(s) contents - Specify minimum versions of common package managers - Remove sharp.vendor runtime API as no-longer relevant - Update installation docs and issue templates
lovell · Sep 25, 2023 · 428e71f · 428e71f
1 parent 0f8bb91
commit 428e71f
Show file tree

Hide file tree

Showing 42 changed files with 821 additions and 6,509 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -36,6 +36,12 @@ jobs:
           sudo docker exec sharp sh -c "apt-get update && apt-get install -y nodejs"
       - run: sudo docker exec sharp sh -c "npm install --build-from-source"
       - run: sudo docker exec sharp sh -c "npm test"
+      - run: |
+          sudo docker exec sharp sh -c "npm run package-from-local-build"
+          sudo docker exec sharp sh -c "npm pkg set \"optionalDependencies.@sharpen/sharp-linux-arm64=file:./npm/linux-arm64\""
+          sudo docker exec sharp sh -c "npm run clean"
+          sudo docker exec sharp sh -c "npm install --ignore-scripts"
+          sudo docker exec sharp sh -c "npm test"
       - run: "[[ -n $CIRCLE_TAG ]] && sudo docker exec --env prebuild_upload sharp sh -c \"npx prebuild --upload=$prebuild_upload\" || true"
   linux-arm64-glibc-node-20:
     resource_class: arm.medium
@@ -54,6 +60,12 @@ jobs:
           sudo docker cp . sharp:/mnt/sharp/.
       - run: sudo docker exec sharp sh -c "npm install --build-from-source"
       - run: sudo docker exec sharp sh -c "npm test"
+      - run: |
+          sudo docker exec sharp sh -c "npm run package-from-local-build"
+          sudo docker exec sharp sh -c "npm pkg set \"optionalDependencies.@sharpen/sharp-linux-arm64=file:./npm/linux-arm64\""
+          sudo docker exec sharp sh -c "npm run clean"
+          sudo docker exec sharp sh -c "npm install --ignore-scripts"
+          sudo docker exec sharp sh -c "npm test"
   linux-arm64-musl-node-18:
     resource_class: arm.medium
     machine:
@@ -65,6 +77,12 @@ jobs:
           sudo docker exec sharp sh -c "apk add build-base git python3 font-noto --update-cache"
       - run: sudo docker exec sharp sh -c "npm install --build-from-source"
       - run: sudo docker exec sharp sh -c "npm test"
+      - run: |
+          sudo docker exec sharp sh -c "npm run package-from-local-build"
+          sudo docker exec sharp sh -c "npm pkg set \"optionalDependencies.@sharpen/sharp-linuxmusl-arm64=file:./npm/linuxmusl-arm64\""
+          sudo docker exec sharp sh -c "npm run clean"
+          sudo docker exec sharp sh -c "npm install --ignore-scripts"
+          sudo docker exec sharp sh -c "npm test"
       - run: "[[ -n $CIRCLE_TAG ]] && sudo docker exec --env prebuild_upload sharp sh -c \"npx prebuild --upload=$prebuild_upload\" || true"
   linux-arm64-musl-node-20:
     resource_class: arm.medium
@@ -79,3 +97,9 @@ jobs:
           sudo docker cp . sharp:/mnt/sharp/.
       - run: sudo docker exec sharp sh -c "npm install --build-from-source"
       - run: sudo docker exec sharp sh -c "npm test"
+      - run: |
+          sudo docker exec sharp sh -c "npm run package-from-local-build"
+          sudo docker exec sharp sh -c "npm pkg set \"optionalDependencies.@sharpen/sharp-linuxmusl-arm64=file:./npm/linuxmusl-arm64\""
+          sudo docker exec sharp sh -c "npm run clean"
+          sudo docker exec sharp sh -c "npm install --ignore-scripts"
+          sudo docker exec sharp sh -c "npm test"
diff --git a/.github/ISSUE_TEMPLATE/installation.md b/.github/ISSUE_TEMPLATE/installation.md
@@ -12,7 +12,6 @@ labels: installation
 <!-- Please place an [x] in the box to confirm. -->
 
 - [ ] I have read the [documentation relating to installation](https://sharp.pixelplumbing.com/install).
-- [ ] I have ensured that the architecture and platform of Node.js used for `npm install` is the same as the architecture and platform of Node.js used at runtime.
 
 ### Are you using the latest version of sharp?
 
@@ -24,13 +23,23 @@ If you cannot confirm this, please upgrade to the latest version and try again b
 
 If you are using another package which depends on a version of `sharp` that is not the latest, please open an issue against that package instead.
 
-### Is this a problem with filesystem permissions?
+### Are you using a supported runtime?
 
-If you are using npm v6 or earlier and installing as a `root` or `sudo` user, have you tried with the `npm install --unsafe-perm` flag?
+<!-- Please place an [x] in the relevant box to confirm. -->
 
-If you are using npm v7 or later, does the user running `npm install` own the directory it is run in?
+- [ ] I am using Node.js 18 with a version >= 18.17.0
+- [ ] I am using Node.js 20 with a version >= 20.3.0
+- [ ] I am using Node.js 21 or later
 
-If you are using the `ignore-scripts` feature of `npm`, have you tried with the `npm install --ignore-scripts=false` flag?
+### Are you using a supported package manager?
+
+<!-- Please place an [x] in the relevant box to confirm. -->
+
+- [ ] I am using npm >= 9.6.5
+- [ ] I am using yarn >= 3.2.0
+- [ ] I am using pnpm >= 7.1.0
+
+If you cannot confirm this, please upgrade to the latest version and try again before opening an issue.
 
 ### What is the complete output of running `npm install --verbose --foreground-scripts sharp` in an empty directory?
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -37,6 +37,7 @@ jobs:
           - os: macos-11
             nodejs_arch: x64
             nodejs_version: "^18.17.0"
+            nodejs_version_major: 18
             platform: darwin-x64
             prebuild: true
           - os: macos-11
@@ -98,13 +99,17 @@ jobs:
           architecture: ${{ matrix.nodejs_arch }}
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Fix working directory ownership
-        if: matrix.container
-        run: chown root.root .
       - name: Install
         run: npm install --build-from-source
       - name: Test
         run: npm test
+      - name: Test packaging
+        run: |
+          npm run package-from-local-build
+          npm pkg set "optionalDependencies.@sharpen/sharp-${{ matrix.platform }}=file:./npm/${{ matrix.platform }}"
+          npm run clean
+          npm install --ignore-scripts
+          npm test
       - name: Prebuild
         if: matrix.prebuild && startsWith(github.ref, 'refs/tags/')
         env:
@@ -130,9 +135,13 @@ jobs:
             mkdir /opt/nodejs
             curl --silent https://unofficial-builds.nodejs.org/download/release/v${nodejs_version}/node-v${nodejs_version}-linux-armv6l.tar.xz | tar xJC /opt/nodejs --strip-components=1
             export PATH=$PATH:/opt/nodejs/bin
-            chown root.root .
             npm install --build-from-source
             npx mocha --no-config --spec=test/unit/io.js
+            npm run package-from-local-build
+            npm pkg set "optionalDependencies.@sharpen/sharp-linux-arm=file:./npm/linux-arm"
+            npm run clean
+            npm install --ignore-scripts
+            npx mocha --no-config --spec=test/unit/io.js
             [[ -n $prebuild_upload ]] && npx prebuild || true
   macstadium-runner:
     permissions:
@@ -167,6 +176,13 @@ jobs:
         run: npm install --build-from-source
       - name: Test
         run: npm test
+      - name: Test packaging
+        run: |
+          npm run package-from-local-build
+          npm pkg set "optionalDependencies.@sharpen/sharp-${{ matrix.platform }}=file:./npm/${{ matrix.platform }}"
+          npm run clean
+          npm install --ignore-scripts
+          npm test
       - name: Prebuild
         if: matrix.prebuild && startsWith(github.ref, 'refs/tags/')
         env:

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,8 @@
 build
 node_modules
 /coverage
+npm/*/*
+!npm/*/package.json
 test/bench/node_modules
 test/fixtures/output*
 test/fixtures/vips-properties.xml
@@ -9,7 +11,6 @@ test/saliency/report.json
 test/saliency/Image*
 test/saliency/[Uu]serData*
 !test/saliency/userData.js
-vendor
 .gitattributes
 .DS_Store
 .nyc_output

diff --git a/.prebuildrc b/.prebuildrc
@@ -1,6 +1,6 @@
 {
   "runtime": "napi",
-  "include-regex": "(sharp-.+\\.node|libvips-cpp\\.dll)",
+  "include-regex": "(sharp-.+\\.node|libvips-.+\\.dll|libglib-.+\\.dll|libgobject-.+\\.dll)",
   "prerelease": true,
   "strip": true
 }
diff --git a/binding.gyp b/binding.gyp
@@ -1,11 +1,16 @@
+# Copyright 2013 Lovell Fuller and others.
+# SPDX-License-Identifier: Apache-2.0
+
 {
   'variables': {
     'vips_version': '<!(node -p "require(\'./lib/libvips\').minimumLibvipsVersion")',
-    'platform_and_arch': '<!(node -p "require(\'./lib/platform\')()")',
-    'sharp_vendor_dir': './vendor/<(vips_version)/<(platform_and_arch)'
+    'platform_and_arch': '<!(node -p "require(\'./lib/libvips\').buildPlatformArch()")',
+    'sharp_libvips_include_dir': '<!(node -p "require(\'./lib/libvips\').buildSharpLibvipsIncludeDir()")',
+    'sharp_libvips_cplusplus_dir': '<!(node -p "require(\'./lib/libvips\').buildSharpLibvipsCPlusPlusDir()")',
+    'sharp_libvips_lib_dir': '<!(node -p "require(\'./lib/libvips\').buildSharpLibvipsLibDir()")'
   },
   'targets': [{
-    'target_name': 'libvips-cpp',
+    'target_name': 'win-libvips-cpp',
     'conditions': [
       ['OS == "win"', {
         # Build libvips C++ binding for Windows due to MSVC std library ABI changes
@@ -15,19 +20,19 @@
           '_ALLOW_KEYWORD_MACROS'
         ],
         'sources': [
-          'src/libvips/cplusplus/VConnection.cpp',
-          'src/libvips/cplusplus/VError.cpp',
-          'src/libvips/cplusplus/VImage.cpp',
-          'src/libvips/cplusplus/VInterpolate.cpp',
-          'src/libvips/cplusplus/VRegion.cpp'
+          '<(sharp_libvips_cplusplus_dir)/VConnection.cpp',
+          '<(sharp_libvips_cplusplus_dir)/VError.cpp',
+          '<(sharp_libvips_cplusplus_dir)/VImage.cpp',
+          '<(sharp_libvips_cplusplus_dir)/VInterpolate.cpp',
+          '<(sharp_libvips_cplusplus_dir)/VRegion.cpp'
         ],
         'include_dirs': [
-          '<(sharp_vendor_dir)/include',
-          '<(sharp_vendor_dir)/include/glib-2.0',
-          '<(sharp_vendor_dir)/lib/glib-2.0/include'
+          '<(sharp_libvips_include_dir)',
+          '<(sharp_libvips_include_dir)/glib-2.0',
+          '<(sharp_libvips_lib_dir)/glib-2.0/include'
         ],
         'link_settings': {
-          'library_dirs': ['<(sharp_vendor_dir)/lib'],
+          'library_dirs': ['<(sharp_libvips_lib_dir)'],
           'libraries': [
             'libvips.lib',
             'libglib-2.0.lib',
@@ -76,10 +81,9 @@
     ],
     'dependencies': [
       '<!(node -p "require(\'node-addon-api\').gyp")',
-      'libvips-cpp'
+      'win-libvips-cpp'
     ],
     'variables': {
-      'runtime_link%': 'shared',
       'conditions': [
         ['OS != "win"', {
           'pkg_config_path': '<!(node -p "require(\'./lib/libvips\').pkgConfigPath()")',
@@ -106,12 +110,8 @@
       ['use_global_libvips == "true"', {
         # Use pkg-config for include and lib
         'include_dirs': ['<!@(PKG_CONFIG_PATH="<(pkg_config_path)" pkg-config --cflags-only-I vips-cpp vips glib-2.0 | sed s\/-I//g)'],
+        'libraries': ['<!@(PKG_CONFIG_PATH="<(pkg_config_path)" pkg-config --libs vips-cpp)'],
         'conditions': [
-          ['runtime_link == "static"', {
-            'libraries': ['<!@(PKG_CONFIG_PATH="<(pkg_config_path)" pkg-config --libs --static vips-cpp)']
-          }, {
-            'libraries': ['<!@(PKG_CONFIG_PATH="<(pkg_config_path)" pkg-config --libs vips-cpp)']
-          }],
           ['OS == "linux"', {
             'defines': [
               # Inspect libvips-cpp.so to determine which C++11 ABI version was used and set _GLIBCXX_USE_CXX11_ABI accordingly. This is quite horrible.
@@ -122,9 +122,9 @@
       }, {
         # Use pre-built libvips stored locally within node_modules
         'include_dirs': [
-          '<(sharp_vendor_dir)/include',
-          '<(sharp_vendor_dir)/include/glib-2.0',
-          '<(sharp_vendor_dir)/lib/glib-2.0/include'
+          '<(sharp_libvips_include_dir)',
+          '<(sharp_libvips_include_dir)/glib-2.0',
+          '<(sharp_libvips_lib_dir)/glib-2.0/include'
         ],
         'conditions': [
           ['OS == "win"', {
@@ -133,7 +133,7 @@
               '_FILE_OFFSET_BITS=64'
             ],
             'link_settings': {
-              'library_dirs': ['<(sharp_vendor_dir)/lib'],
+              'library_dirs': ['<(sharp_libvips_lib_dir)'],
               'libraries': [
                 'libvips.lib',
                 'libglib-2.0.lib',
@@ -143,15 +143,19 @@
           }],
           ['OS == "mac"', {
             'link_settings': {
-              'library_dirs': ['../<(sharp_vendor_dir)/lib'],
+              'library_dirs': [
+                '<(sharp_libvips_lib_dir)'
+              ],
               'libraries': [
                 'libvips-cpp.42.dylib'
               ]
             },
             'xcode_settings': {
               'OTHER_LDFLAGS': [
                 # Ensure runtime linking is relative to sharp.node
-                '-Wl,-rpath,\'@loader_path/../../<(sharp_vendor_dir)/lib\''
+                '-Wl,-rpath,\'@loader_path/../../sharp-libvips-<(platform_and_arch)/lib\'',
+                '-Wl,-rpath,\'@loader_path/../../node_modules/@sharpen/sharp-libvips-<(platform_and_arch)/lib\'',
+                '-Wl,-rpath,\'@loader_path/../../../node_modules/@sharpen/sharp-libvips-<(platform_and_arch)/lib\''
               ]
             }
           }],
@@ -160,13 +164,19 @@
               '_GLIBCXX_USE_CXX11_ABI=1'
             ],
             'link_settings': {
-              'library_dirs': ['../<(sharp_vendor_dir)/lib'],
+              'library_dirs': [
+                '<(sharp_libvips_lib_dir)'
+              ],
               'libraries': [
                 '-l:libvips-cpp.so.42'
               ],
               'ldflags': [
                 # Ensure runtime linking is relative to sharp.node
-                '-Wl,-s -Wl,--disable-new-dtags -Wl,-rpath=\'$$ORIGIN/../../<(sharp_vendor_dir)/lib\''
+                '-Wl,-s',
+                '-Wl,--disable-new-dtags',
+                '-Wl,-rpath=\'$$ORIGIN/../../sharp-libvips-<(platform_and_arch)/lib\'',
+                '-Wl,-rpath=\'$$ORIGIN/../../node_modules/@sharpen/sharp-libvips-<(platform_and_arch)/lib\'',
+                '-Wl,-rpath=\'$$ORIGIN/../../../node_modules/@sharpen/sharp-libvips-<(platform_and_arch)/lib\''
               ]
             }
           }]
@@ -232,5 +242,23 @@
         ]
       }
     },
+  }, {
+    'target_name': 'win-copy-dlls',
+    'type': 'none',
+    'dependencies': [
+      'sharp-<(platform_and_arch)'
+    ],
+    'conditions': [
+      ['OS == "win"', {
+        'copies': [{
+          'destination': 'build/Release',
+          'files': [
+            '<(sharp_libvips_lib_dir)/libvips-42.dll',
+            '<(sharp_libvips_lib_dir)/libglib-2.0-0.dll',
+            '<(sharp_libvips_lib_dir)/libgobject-2.0-0.dll'
+          ]
+        }]
+      }]
+    ]
   }]
 }