From 5be054f05f4d85a093d618cc6b92c39c9f59a7eb Mon Sep 17 00:00:00 2001 From: Stainless Bot Date: Thu, 11 Apr 2024 16:12:28 +0000 Subject: [PATCH] chore(internal): improve ecosystem tests --- .gitignore | 5 + .prettierignore | 2 +- ecosystem-tests/cli.ts | 226 +++++++++++++++++++++++---- ecosystem-tests/deno/deno.jsonc | 4 + ecosystem-tests/deno/deno.lock | 32 ++-- ecosystem-tests/deno/import_map.json | 6 - ecosystem-tests/deno/main_test.ts | 5 +- 7 files changed, 229 insertions(+), 51 deletions(-) delete mode 100644 ecosystem-tests/deno/import_map.json diff --git a/.gitignore b/.gitignore index 58b3944a1..31b12ac63 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ dist /deno /*.tgz .idea/ +tmp +.pack +ecosystem-tests/deno/package.json +ecosystem-tests/*/openai.tgz + diff --git a/.prettierignore b/.prettierignore index fc6160fb1..3548c5af9 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,5 @@ CHANGELOG.md -/ecosystem-tests +/ecosystem-tests/*/** /node_modules /deno diff --git a/ecosystem-tests/cli.ts b/ecosystem-tests/cli.ts index c84c479d4..a3c1f27a4 100644 --- a/ecosystem-tests/cli.ts +++ b/ecosystem-tests/cli.ts @@ -5,16 +5,19 @@ import assert from 'assert'; import path from 'path'; const TAR_NAME = 'openai.tgz'; -const PACK_FILE = `.pack/${TAR_NAME}`; +const PACK_FOLDER = '.pack'; +const PACK_FILE = `${PACK_FOLDER}/${TAR_NAME}`; const IS_CI = Boolean(process.env['CI'] && process.env['CI'] !== 'false'); async function defaultNodeRunner() { await installPackage(); await run('npm', ['run', 'tsc']); - if (state.live) await run('npm', ['test']); + if (state.live) { + await run('npm', ['test']); + } } -const projects = { +const projectRunners = { 'node-ts-cjs': defaultNodeRunner, 'node-ts-cjs-web': defaultNodeRunner, 'node-ts-cjs-auto': defaultNodeRunner, @@ -76,30 +79,17 @@ const projects = { } }, deno: async () => { + // we don't need to explicitly install the package here + // because our deno setup relies on `rootDir/deno` to exist + // which is an artifact produced from our build process await run('deno', ['task', 'install']); - await installPackage(); - const packFile = getPackFile(); - - const openaiDir = path.resolve( - process.cwd(), - 'node_modules', - '.deno', - 'openai@3.3.0', - 'node_modules', - 'openai', - ); - - await run('sh', ['-c', 'rm -rf *'], { cwd: openaiDir, stdio: 'inherit' }); - await run('tar', ['xzf', path.resolve(packFile)], { cwd: openaiDir, stdio: 'inherit' }); - await run('sh', ['-c', 'mv package/* .'], { cwd: openaiDir, stdio: 'inherit' }); - await run('sh', ['-c', 'rm -rf package'], { cwd: openaiDir, stdio: 'inherit' }); - await run('deno', ['task', 'check']); + if (state.live) await run('deno', ['task', 'test']); }, }; -const projectNames = Object.keys(projects) as Array; +let projectNames = Object.keys(projectRunners) as Array; const projectNamesSet = new Set(projectNames); function parseArgs() { @@ -118,6 +108,11 @@ function parseArgs() { type: 'boolean', default: false, }, + skip: { + type: 'array', + default: [], + description: 'Skip one or more projects. Separate project names with a space.', + }, skipPack: { type: 'boolean', default: false, @@ -156,6 +151,10 @@ function parseArgs() { default: false, description: 'run all projects in parallel (jobs = # projects)', }, + noCleanup: { + type: 'boolean', + default: false, + }, }) .help().argv; } @@ -165,9 +164,32 @@ type Args = Awaited>; let state: Args & { rootDir: string }; async function main() { + if (!process.env['OPENAI_API_KEY']) { + console.error(`Error: The environment variable OPENAI_API_KEY must be set. Run the command + $echo 'OPENAI_API_KEY = "'"\${OPENAI_API_KEY}"'"' >> ecosystem-tests/cloudflare-worker/wrangler.toml`); + process.exit(0); + } + const args = (await parseArgs()) as Args; console.error(`args:`, args); + // Some projects, e.g. Deno can be slow to run, so offer the option to skip them. Example: + // --skip=deno node-ts-cjs + if (args.skip.length > 0) { + args.skip.forEach((projectName, idx) => { + // Ensure the inputted project name is lower case + args.skip[idx] = (projectName + '').toLowerCase(); + }); + + projectNames = projectNames.filter((projectName) => (args.skip as string[]).indexOf(projectName) < 0); + + args.skip.forEach((projectName) => { + projectNamesSet.delete(projectName as any); + }); + } + + const tmpFolderPath = path.resolve(process.cwd(), 'tmp'); + const rootDir = await packageDir(); console.error(`rootDir:`, rootDir); @@ -191,8 +213,63 @@ async function main() { const failed: typeof projectNames = []; + let cleanupWasRun = false; + + // Cleanup the various artifacts created as part of executing this script + async function runCleanup() { + if (cleanupWasRun) { + return; + } + cleanupWasRun = true; + + // Restore the original files in the ecosystem-tests folders from before + // npm install was run + await fileCache.restoreFiles(tmpFolderPath); + + const packFolderPath = path.join(process.cwd(), PACK_FOLDER); + + try { + // Clean up the .pack folder if this was the process that created it. + await fs.unlink(PACK_FILE); + await fs.rmdir(packFolderPath); + } catch (err) { + console.log('Failed to delete .pack folder', err); + } + + for (let i = 0; i < projectNames.length; i++) { + const projectName = (projectNames as any)[i] as string; + + await defaultNodeCleanup(projectName).catch((err: any) => { + console.error('Error: Cleanup of file artifacts failed for project', projectName, err); + }); + } + } + + async function runCleanupAndExit() { + await runCleanup(); + + process.exit(1); + } + + if (!(await fileExists(tmpFolderPath))) { + await fs.mkdir(tmpFolderPath); + } + let { jobs } = args; - if (args.parallel) jobs = projectsToRun.length; + if (args.parallel) { + jobs = projectsToRun.length; + } + + if (!args.noCleanup) { + // The cleanup code is only executed from the parent script that runs + // multiple projects. + process.on('SIGINT', runCleanupAndExit); + process.on('SIGTERM', runCleanupAndExit); + process.on('exit', runCleanup); + + await fileCache.cacheFiles(tmpFolderPath); + } + if (jobs > 1) { const queue = [...projectsToRun]; const runningProjects = new Set(); @@ -225,7 +302,9 @@ async function main() { [...Array(jobs).keys()].map(async () => { while (queue.length) { const project = queue.shift(); - if (!project) break; + if (!project) { + break; + } // preserve interleaved ordering of writes to stdout/stderr const chunks: { dest: 'stdout' | 'stderr'; data: string | Buffer }[] = []; @@ -238,6 +317,7 @@ async function main() { __filename, project, '--skip-pack', + '--noCleanup', `--retry=${args.retry}`, ...(args.live ? ['--live'] : []), ...(args.verbose ? ['--verbose'] : []), @@ -248,6 +328,7 @@ async function main() { ); child.stdout?.on('data', (data) => chunks.push({ dest: 'stdout', data })); child.stderr?.on('data', (data) => chunks.push({ dest: 'stderr', data })); + await child; } catch (error) { failed.push(project); @@ -255,7 +336,10 @@ async function main() { runningProjects.delete(project); } - if (IS_CI) console.log(`::group::${failed.includes(project) ? '❌' : '✅'} ${project}`); + if (IS_CI) { + console.log(`::group::${failed.includes(project) ? '❌' : '✅'} ${project}`); + } + for (const { data } of chunks) { process.stdout.write(data); } @@ -268,7 +352,7 @@ async function main() { clearProgress(); } else { for (const project of projectsToRun) { - const fn = projects[project]; + const fn = projectRunners[project]; await withChdir(path.join(rootDir, 'ecosystem-tests', project), async () => { console.error('\n'); @@ -294,6 +378,10 @@ async function main() { } } + if (!args.noCleanup) { + await runCleanup(); + } + if (failed.length) { console.error(`${failed.length} project(s) failed - ${failed.join(', ')}`); process.exit(1); @@ -340,10 +428,15 @@ async function buildPackage() { return; } - if (!(await pathExists('.pack'))) { - await fs.mkdir('.pack'); + if (!(await pathExists(PACK_FOLDER))) { + await fs.mkdir(PACK_FOLDER); } + // Run our build script to ensure all of our build artifacts are up to date. + // This matters the most for deno as it directly relies on build artifacts + // instead of the pack file + await run('yarn', ['build']); + const proc = await run('npm', ['pack', '--ignore-scripts', '--json'], { cwd: path.join(process.cwd(), 'dist'), alwaysPipe: true, @@ -366,6 +459,11 @@ async function installPackage() { return; } + try { + // Ensure that there is a clean node_modules folder. + await run('rm', ['-rf', `./node_modules`]); + } catch (err) {} + const packFile = getPackFile(); await fs.copyFile(packFile, `./${TAR_NAME}`); return await run('npm', ['install', '-D', `./${TAR_NAME}`]); @@ -440,6 +538,80 @@ export const packageDir = async (): Promise => { throw new Error('Package directory not found'); }; +// Caches files that are modified by this script, e.g. package.json, +// so that they can be restored when the script either finishes or is +// terminated +const fileCache = (() => { + const filesToCache: Array = ['package.json', 'package-lock.json', 'deno.lock', 'bun.lockb']; + + return { + // Copy existing files from each ecosystem-tests project folder to the ./tmp folder + cacheFiles: async (tmpFolderPath: string) => { + for (let i = 0; i < projectNames.length; i++) { + const projectName = (projectNames as any)[i] as string; + const projectPath = path.resolve(process.cwd(), 'ecosystem-tests', projectName); + + for (let j = 0; j < filesToCache.length; j++) { + const fileName = filesToCache[j] || ''; + + const filePath = path.resolve(projectPath, fileName); + if (await fileExists(filePath)) { + const tmpProjectPath = path.resolve(tmpFolderPath, projectName); + + if (!(await fileExists(tmpProjectPath))) { + await fs.mkdir(tmpProjectPath); + } + await fs.copyFile(filePath, path.resolve(tmpProjectPath, fileName)); + } + } + } + }, + + // Restore the original files to each ecosystem-tests project folder from the ./tmp folder + restoreFiles: async (tmpFolderPath: string) => { + for (let i = 0; i < projectNames.length; i++) { + const projectName = (projectNames as any)[i] as string; + + const projectPath = path.resolve(process.cwd(), 'ecosystem-tests', projectName); + const tmpProjectPath = path.resolve(tmpFolderPath, projectName); + + for (let j = 0; j < filesToCache.length; j++) { + const fileName = filesToCache[j] || ''; + + const filePath = path.resolve(tmpProjectPath, fileName); + if (await fileExists(filePath)) { + await fs.rename(filePath, path.resolve(projectPath, fileName)); + } + } + await fs.rmdir(tmpProjectPath); + } + }, + }; +})(); + +async function defaultNodeCleanup(projectName: string) { + try { + const projectPath = path.resolve(process.cwd(), 'ecosystem-tests', projectName); + + const packFilePath = path.resolve(projectPath, TAR_NAME); + + if (await fileExists(packFilePath)) { + await fs.unlink(packFilePath); + } + } catch (err) { + console.error('Cleanup failed for project', projectName, err); + } +} + +async function fileExists(filePath: string) { + try { + await fs.stat(filePath); + return true; + } catch { + return false; + } +} + main().catch((err) => { console.error(err); process.exit(1); diff --git a/ecosystem-tests/deno/deno.jsonc b/ecosystem-tests/deno/deno.jsonc index ba78e9d30..7de05f2ba 100644 --- a/ecosystem-tests/deno/deno.jsonc +++ b/ecosystem-tests/deno/deno.jsonc @@ -3,5 +3,9 @@ "install": "deno install --node-modules-dir main_test.ts -f", "check": "deno lint && deno check main_test.ts", "test": "deno test --allow-env --allow-net --allow-read --node-modules-dir" + }, + "imports": { + "openai": "../../deno/mod.ts", + "openai/": "../../deno/" } } diff --git a/ecosystem-tests/deno/deno.lock b/ecosystem-tests/deno/deno.lock index 17a25fcbc..aa22a1427 100644 --- a/ecosystem-tests/deno/deno.lock +++ b/ecosystem-tests/deno/deno.lock @@ -1,20 +1,14 @@ { - "version": "2", - "remote": { - "https://deno.land/std@0.192.0/fmt/colors.ts": "d67e3cd9f472535241a8e410d33423980bec45047e343577554d3356e1f0ef4e", - "https://deno.land/std@0.192.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", - "https://deno.land/std@0.192.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", - "https://deno.land/std@0.192.0/testing/asserts.ts": "e16d98b4d73ffc4ed498d717307a12500ae4f2cbe668f1a215632d19fcffc22f" - }, - "npm": { + "version": "3", + "packages": { "specifiers": { - "@types/node@^20.3.1": "@types/node@20.3.1", - "node-fetch@^3.0.0": "node-fetch@3.3.1", - "openai": "openai@3.3.0", - "ts-node@^10.9.1": "ts-node@10.9.1_@types+node@20.3.1_typescript@5.1.3", - "typescript@^5.1.3": "typescript@5.1.3" + "npm:@types/node@^20.3.1": "npm:@types/node@20.3.1", + "npm:node-fetch@^3.0.0": "npm:node-fetch@3.3.1", + "npm:openai": "npm:openai@3.3.0", + "npm:ts-node@^10.9.1": "npm:ts-node@10.9.1_@types+node@20.3.1_typescript@5.1.3", + "npm:typescript@^5.1.3": "npm:typescript@5.1.3" }, - "packages": { + "npm": { "@cspotcode/source-map-support@0.8.1": { "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", "dependencies": { @@ -195,5 +189,15 @@ "dependencies": {} } } + }, + "redirects": { + "https://deno.land/x/fastest_levenshtein/mod.ts": "https://deno.land/x/fastest_levenshtein@1.0.10/mod.ts" + }, + "remote": { + "https://deno.land/std@0.192.0/fmt/colors.ts": "d67e3cd9f472535241a8e410d33423980bec45047e343577554d3356e1f0ef4e", + "https://deno.land/std@0.192.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", + "https://deno.land/std@0.192.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", + "https://deno.land/std@0.192.0/testing/asserts.ts": "e16d98b4d73ffc4ed498d717307a12500ae4f2cbe668f1a215632d19fcffc22f", + "https://deno.land/x/fastest_levenshtein@1.0.10/mod.ts": "aea49d54b6bb37082b2377da2ea068331da07b2a515621d8eff97538b7157b40" } } diff --git a/ecosystem-tests/deno/import_map.json b/ecosystem-tests/deno/import_map.json deleted file mode 100644 index 941f5396b..000000000 --- a/ecosystem-tests/deno/import_map.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "imports": { - "/": "./", - "./": "./" - } -} diff --git a/ecosystem-tests/deno/main_test.ts b/ecosystem-tests/deno/main_test.ts index b841b4053..b27c9079b 100644 --- a/ecosystem-tests/deno/main_test.ts +++ b/ecosystem-tests/deno/main_test.ts @@ -1,7 +1,6 @@ import { assertEquals, AssertionError } from 'https://deno.land/std@0.192.0/testing/asserts.ts'; -import OpenAI, { toFile } from 'npm:openai@3.3.0'; import { distance } from 'https://deno.land/x/fastest_levenshtein/mod.ts'; -import { ChatCompletion } from 'npm:openai@3.3.0/resources/chat/completions'; +import OpenAI, { toFile } from 'openai'; const url = 'https://audio-samples.github.io/samples/mp3/blizzard_biased/sample-1.mp3'; const filename = 'sample-1.mp3'; @@ -66,7 +65,7 @@ Deno.test(async function rawResponse() { offset += chunk.length; } - const json: ChatCompletion = JSON.parse(new TextDecoder().decode(merged)); + const json: OpenAI.ChatCompletion = JSON.parse(new TextDecoder().decode(merged)); assertSimilar(json.choices[0]?.message.content || '', 'This is a test', 10); });