From fd0182b03f6373ed1f4ffccf2d874f20026b880e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 31 Jan 2024 13:05:51 -0800 Subject: [PATCH 1/7] Update sharp special case for pnpm --- src/utils/special-cases.ts | 25 ++++++- test/integration.test.js | 122 +++++++++++++++++++++------------ test/integration/sharp-pnpm.js | 9 +++ test/integration/sharp.js | 5 -- 4 files changed, 113 insertions(+), 48 deletions(-) create mode 100644 test/integration/sharp-pnpm.js diff --git a/src/utils/special-cases.ts b/src/utils/special-cases.ts index c5d0f9f3..5bd96087 100644 --- a/src/utils/special-cases.ts +++ b/src/utils/special-cases.ts @@ -109,13 +109,36 @@ const specialCases: Record void> = { emitAsset(resolve(id.replace('index.js', 'preload.js'))); } }, - 'sharp' ({ id, emitAssetDirectory }) { + sharp: async ({ id, emitAssetDirectory, job }) => { if (id.endsWith('sharp/lib/index.js')) { const file = resolve(id, '..', '..', 'package.json'); const pkg = JSON.parse(readFileSync(file, 'utf8')); for (const dep of Object.keys(pkg.optionalDependencies || {})) { const dir = resolve(id, '..', '..', '..', dep); emitAssetDirectory(dir); + + try { + const file = resolve(dir, 'package.json'); + const pkg = JSON.parse(readFileSync(file, 'utf8')); + for (const innerDep of Object.keys(pkg.optionalDependencies || {})) { + const innerDir = resolve( + await job.realpath(dir), + '..', + '..', + innerDep + ); + emitAssetDirectory(innerDir); + } + } catch (err: unknown) { + if ( + err && + typeof err === 'object' && + 'code' in err && + err.code !== 'ENOENT' + ) { + throw err; + } + } } } }, diff --git a/test/integration.test.js b/test/integration.test.js index e7d9cdcd..5236eca0 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -3,68 +3,106 @@ const path = require('path'); const { nodeFileTrace } = require('../out/node-file-trace'); const os = require('os'); const rimraf = require('rimraf'); -const { readFile, writeFile, readlink, symlink } = promises; -const { fork } = require('child_process'); +const { readFile, writeFile, readlink, symlink, mkdtemp, copyFile } = promises; +const { fork, exec: execOrig } = require('child_process'); + +const exec = require('util').promisify(execOrig); jest.setTimeout(200_000); const integrationDir = `${__dirname}${path.sep}integration`; for (const integrationTest of readdirSync(integrationDir)) { + let currentIntegrationDir = integrationDir; + it(`should correctly trace and correctly execute ${integrationTest}`, async () => { console.log('Tracing and executing ' + integrationTest); - const nftCache = {} + const nftCache = {}; const fails = integrationTest.endsWith('failure.js'); - const { fileList, reasons, warnings } = await nodeFileTrace([`${integrationDir}/${integrationTest}`], { - log: true, - cache: nftCache, - base: path.resolve(__dirname, '..'), - processCwd: integrationDir, - // ignore other integration tests - ignore: ['test/integration/**'] - }); + + let traceBase = path.resolve(__dirname, '..') + + if (integrationTest === 'sharp-pnpm.js') { + currentIntegrationDir = await mkdtemp(path.join(os.tmpdir(), `nft-${integrationTest}`)); + await copyFile( + path.join(integrationDir, integrationTest), + path.join(currentIntegrationDir, integrationTest) + ); + await writeFile( + path.join(currentIntegrationDir, 'package.json'), + JSON.stringify({ + name: 'sharp-pnpm', + dependencies: { + sharp: 'latest', + }, + packageManager: 'pnpm@8.15.1', + }) + ); + traceBase = currentIntegrationDir + await exec(`corepack enable && pnpm i`, { + cwd: currentIntegrationDir, + stdio: 'inherit', + }); + } + + const { fileList, reasons, warnings } = await nodeFileTrace( + [`${currentIntegrationDir}/${integrationTest}`], + { + log: true, + cache: nftCache, + base: traceBase, + processCwd: currentIntegrationDir, + // ignore other integration tests + ignore: ['test/integration/**'], + } + ); // warnings.forEach(warning => console.warn(warning)); - const randomTmpId = Math.random().toString().slice(2) + const randomTmpId = Math.random().toString().slice(2); const tmpdir = path.resolve(os.tmpdir(), `node-file-trace-${randomTmpId}`); rimraf.sync(tmpdir); mkdirSync(tmpdir); - await Promise.all([...fileList].map(async file => { - const inPath = path.resolve(__dirname, '..', file); - const outPath = path.resolve(tmpdir, file); - try { - var symlinkPath = await readlink(inPath); - } - catch (e) { - if (e.code !== 'EINVAL' && e.code !== 'UNKNOWN') throw e; - } - mkdirSync(path.dirname(outPath), { recursive: true }); - if (symlinkPath) { - await symlink(symlinkPath, outPath); - } - else { - await writeFile(outPath, await readFile(inPath), { mode: 0o777 }); - } - })); - const testFile = path.join(tmpdir, 'test', 'integration', integrationTest); + + await Promise.all( + [...fileList].map(async (file) => { + const inPath = path.resolve(traceBase, file); + const outPath = path.resolve(tmpdir, file); + try { + var symlinkPath = await readlink(inPath); + } catch (e) { + if (e.code !== 'EINVAL' && e.code !== 'UNKNOWN') throw e; + } + mkdirSync(path.dirname(outPath), { recursive: true }); + if (symlinkPath) { + await symlink(symlinkPath, outPath); + } else { + await writeFile(outPath, await readFile(inPath), { mode: 0o777 }); + } + }) + ); + const testFile = path.join(tmpdir, path.relative(traceBase, currentIntegrationDir), integrationTest); + const ps = fork(testFile, { - stdio: fails ? 'pipe' : 'inherit' + stdio: fails ? 'pipe' : 'inherit', }); - const code = await new Promise(resolve => ps.on('close', resolve)); + const code = await new Promise((resolve) => ps.on('close', resolve)); expect(code).toBe(fails ? 1 : 0); rimraf.sync(tmpdir); - + // TODO: ensure analysis cache is safe for below case // seems this fails with cache since < 0.14.0 if (integrationTest !== 'browserify-middleware.js') { - const cachedResult = await nodeFileTrace([`${integrationDir}/${integrationTest}`], { - log: true, - cache: nftCache, - base: path.resolve(__dirname, '..'), - processCwd: integrationDir, - // ignore other integration tests - ignore: ['test/integration/**'] - }); - expect([...cachedResult.fileList].sort()).toEqual([...fileList].sort()) + const cachedResult = await nodeFileTrace( + [`${currentIntegrationDir}/${integrationTest}`], + { + log: true, + cache: nftCache, + base: traceBase, + processCwd: currentIntegrationDir, + // ignore other integration tests + ignore: ['test/integration/**'], + } + ); + expect([...cachedResult.fileList].sort()).toEqual([...fileList].sort()); } }); } diff --git a/test/integration/sharp-pnpm.js b/test/integration/sharp-pnpm.js new file mode 100644 index 00000000..71f8e5c7 --- /dev/null +++ b/test/integration/sharp-pnpm.js @@ -0,0 +1,9 @@ +const sharp = require('sharp'); + +const roundedCorners = Buffer.from( + '' +); + +sharp(roundedCorners) + .resize(200, 200) + .png().toBuffer(); diff --git a/test/integration/sharp.js b/test/integration/sharp.js index 8ed7491a..71f8e5c7 100644 --- a/test/integration/sharp.js +++ b/test/integration/sharp.js @@ -1,5 +1,4 @@ const sharp = require('sharp'); -const path = require('path'); const roundedCorners = Buffer.from( '' @@ -8,7 +7,3 @@ const roundedCorners = Buffer.from( sharp(roundedCorners) .resize(200, 200) .png().toBuffer(); - -sharp(path.resolve(__dirname, '../fixtures/img.jpg')) - .resize({ width: 100, height: 100 }) - .jpeg().toBuffer(); From b0a563a0459813af992a68374387720ca58f6b01 Mon Sep 17 00:00:00 2001 From: Steven Date: Wed, 31 Jan 2024 17:32:53 -0500 Subject: [PATCH 2/7] try with `pnpm@8.14.3` --- test/integration.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration.test.js b/test/integration.test.js index 5236eca0..985744e0 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -35,7 +35,7 @@ for (const integrationTest of readdirSync(integrationDir)) { dependencies: { sharp: 'latest', }, - packageManager: 'pnpm@8.15.1', + packageManager: 'pnpm@8.14.3', }) ); traceBase = currentIntegrationDir From f51af6147659e744fa81f810dc330acd0d7a5234 Mon Sep 17 00:00:00 2001 From: Steven Date: Wed, 31 Jan 2024 17:57:38 -0500 Subject: [PATCH 3/7] try with `mkdirSync` --- test/integration.test.js | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/test/integration.test.js b/test/integration.test.js index 985744e0..e9785228 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -3,7 +3,7 @@ const path = require('path'); const { nodeFileTrace } = require('../out/node-file-trace'); const os = require('os'); const rimraf = require('rimraf'); -const { readFile, writeFile, readlink, symlink, mkdtemp, copyFile } = promises; +const { readFile, writeFile, readlink, symlink, copyFile } = promises; const { fork, exec: execOrig } = require('child_process'); const exec = require('util').promisify(execOrig); @@ -18,31 +18,25 @@ for (const integrationTest of readdirSync(integrationDir)) { it(`should correctly trace and correctly execute ${integrationTest}`, async () => { console.log('Tracing and executing ' + integrationTest); const nftCache = {}; + const rand = Math.random().toString().slice(2); const fails = integrationTest.endsWith('failure.js'); let traceBase = path.resolve(__dirname, '..') if (integrationTest === 'sharp-pnpm.js') { - currentIntegrationDir = await mkdtemp(path.join(os.tmpdir(), `nft-${integrationTest}`)); + currentIntegrationDir = path.resolve(os.tmpdir(), `node-file-trace-${integrationTest}-${rand}`); + rimraf.sync(currentIntegrationDir); + mkdirSync(currentIntegrationDir); await copyFile( path.join(integrationDir, integrationTest), path.join(currentIntegrationDir, integrationTest) ); await writeFile( path.join(currentIntegrationDir, 'package.json'), - JSON.stringify({ - name: 'sharp-pnpm', - dependencies: { - sharp: 'latest', - }, - packageManager: 'pnpm@8.14.3', - }) + JSON.stringify({ packageManager: 'pnpm@8.14.3', dependencies: { sharp: '0.33.2' } }) ); traceBase = currentIntegrationDir - await exec(`corepack enable && pnpm i`, { - cwd: currentIntegrationDir, - stdio: 'inherit', - }); + await exec(`corepack enable && pnpm i`, { cwd: currentIntegrationDir, stdio: 'inherit' }); } const { fileList, reasons, warnings } = await nodeFileTrace( @@ -57,8 +51,7 @@ for (const integrationTest of readdirSync(integrationDir)) { } ); // warnings.forEach(warning => console.warn(warning)); - const randomTmpId = Math.random().toString().slice(2); - const tmpdir = path.resolve(os.tmpdir(), `node-file-trace-${randomTmpId}`); + const tmpdir = path.resolve(os.tmpdir(), `node-file-trace-${rand}`); rimraf.sync(tmpdir); mkdirSync(tmpdir); From 42f66ed555d49db29421899d651620fc44a1be12 Mon Sep 17 00:00:00 2001 From: Steven Date: Wed, 31 Jan 2024 18:46:30 -0500 Subject: [PATCH 4/7] swap corepack for npx --- test/integration.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration.test.js b/test/integration.test.js index e9785228..1043f3ac 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -33,10 +33,10 @@ for (const integrationTest of readdirSync(integrationDir)) { ); await writeFile( path.join(currentIntegrationDir, 'package.json'), - JSON.stringify({ packageManager: 'pnpm@8.14.3', dependencies: { sharp: '0.33.2' } }) + JSON.stringify({ dependencies: { sharp: '0.33.2' } }) ); traceBase = currentIntegrationDir - await exec(`corepack enable && pnpm i`, { cwd: currentIntegrationDir, stdio: 'inherit' }); + await exec(`npx pnpm@8.14.3 install`, { cwd: currentIntegrationDir, stdio: 'inherit' }); } const { fileList, reasons, warnings } = await nodeFileTrace( From e6bec08692c4c90a8c496d2878b73f7475bc3a16 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 1 Feb 2024 14:21:50 -0500 Subject: [PATCH 5/7] Revert "swap corepack for npx" This reverts commit 42f66ed555d49db29421899d651620fc44a1be12. --- test/integration.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration.test.js b/test/integration.test.js index 1043f3ac..e9785228 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -33,10 +33,10 @@ for (const integrationTest of readdirSync(integrationDir)) { ); await writeFile( path.join(currentIntegrationDir, 'package.json'), - JSON.stringify({ dependencies: { sharp: '0.33.2' } }) + JSON.stringify({ packageManager: 'pnpm@8.14.3', dependencies: { sharp: '0.33.2' } }) ); traceBase = currentIntegrationDir - await exec(`npx pnpm@8.14.3 install`, { cwd: currentIntegrationDir, stdio: 'inherit' }); + await exec(`corepack enable && pnpm i`, { cwd: currentIntegrationDir, stdio: 'inherit' }); } const { fileList, reasons, warnings } = await nodeFileTrace( From 89ade222785d7fee40cf48fe811e58d32b18fcfb Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 1 Feb 2024 14:26:31 -0500 Subject: [PATCH 6/7] skip node@18 on windows --- test/integration.test.js | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/test/integration.test.js b/test/integration.test.js index e9785228..3a808894 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -20,23 +20,30 @@ for (const integrationTest of readdirSync(integrationDir)) { const nftCache = {}; const rand = Math.random().toString().slice(2); const fails = integrationTest.endsWith('failure.js'); - - let traceBase = path.resolve(__dirname, '..') + let traceBase = path.resolve(__dirname, '..'); if (integrationTest === 'sharp-pnpm.js') { - currentIntegrationDir = path.resolve(os.tmpdir(), `node-file-trace-${integrationTest}-${rand}`); - rimraf.sync(currentIntegrationDir); - mkdirSync(currentIntegrationDir); + if (process.version.startsWith('v18.') && process.platform === 'win32') { + console.log( + 'Skipping sharp-pnpm.js on Node 18 and Windows because of a bug:\n' + + 'https://github.com/nodejs/node/issues/18518' + ); + return; + } + const tmpdir = path.resolve(os.tmpdir(), `node-file-trace-${integrationTest}-${rand}`); + rimraf.sync(tmpdir); + mkdirSync(tmpdir); await copyFile( path.join(integrationDir, integrationTest), - path.join(currentIntegrationDir, integrationTest) + path.join(tmpdir, integrationTest) ); await writeFile( - path.join(currentIntegrationDir, 'package.json'), + path.join(tmpdir, 'package.json'), JSON.stringify({ packageManager: 'pnpm@8.14.3', dependencies: { sharp: '0.33.2' } }) ); - traceBase = currentIntegrationDir - await exec(`corepack enable && pnpm i`, { cwd: currentIntegrationDir, stdio: 'inherit' }); + await exec(`corepack enable && pnpm i`, { cwd: tmpdir, stdio: 'inherit' }); + currentIntegrationDir = tmpdir; + traceBase = tmpdir; } const { fileList, reasons, warnings } = await nodeFileTrace( From 3dd0b6b1a2653e226e9368b8117c86661a07a1e7 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 1 Feb 2024 14:49:13 -0500 Subject: [PATCH 7/7] cleanup --- src/utils/special-cases.ts | 19 +++++-------------- test/integration.test.js | 2 +- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/utils/special-cases.ts b/src/utils/special-cases.ts index 5bd96087..6c5d6bfa 100644 --- a/src/utils/special-cases.ts +++ b/src/utils/special-cases.ts @@ -109,7 +109,7 @@ const specialCases: Record void> = { emitAsset(resolve(id.replace('index.js', 'preload.js'))); } }, - sharp: async ({ id, emitAssetDirectory, job }) => { + 'sharp': async ({ id, emitAssetDirectory, job }) => { if (id.endsWith('sharp/lib/index.js')) { const file = resolve(id, '..', '..', 'package.json'); const pkg = JSON.parse(readFileSync(file, 'utf8')); @@ -121,21 +121,12 @@ const specialCases: Record void> = { const file = resolve(dir, 'package.json'); const pkg = JSON.parse(readFileSync(file, 'utf8')); for (const innerDep of Object.keys(pkg.optionalDependencies || {})) { - const innerDir = resolve( - await job.realpath(dir), - '..', - '..', - innerDep - ); + const innerDir = resolve(await job.realpath(dir), '..', '..', innerDep); emitAssetDirectory(innerDir); } - } catch (err: unknown) { - if ( - err && - typeof err === 'object' && - 'code' in err && - err.code !== 'ENOENT' - ) { + } catch (err: any) { + if (err && err.code !== 'ENOENT') { + console.error(`Error reading "sharp" dependencies from "${dir}/package.json"'`); throw err; } } diff --git a/test/integration.test.js b/test/integration.test.js index 3a808894..a57d40d4 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -25,7 +25,7 @@ for (const integrationTest of readdirSync(integrationDir)) { if (integrationTest === 'sharp-pnpm.js') { if (process.version.startsWith('v18.') && process.platform === 'win32') { console.log( - 'Skipping sharp-pnpm.js on Node 18 and Windows because of a bug:\n' + + 'Skipping sharp-pnpm.js on Node 18 and Windows because of a bug: ' + 'https://github.com/nodejs/node/issues/18518' ); return;