feat: non-breaking ESM support 🤝 (#613)

Closes #608

This adds some esm-vs-commonjs "smartness" to umzug:

1. use `require` for `.cjs` migrations and `import` for `.mjs`
migrations (and their typescript equivalents)
2. use `require` for `.js` migrations _if_ `typeof require.main ===
'object'`, and `import` to `.js` migrations otherwise
3. use the same criteria to create (c)js vs mjs templates when creating
migration files
4. add `"moduleResolution": "Node16"` to tsconfig.lib.json to make sure
`import(filepath)` doesn't get transpiled into
`__importStar(require(filepath))` (see
[here](microsoft/TypeScript#43329 (comment))
and
[here](microsoft/TypeScript#43329 (comment)))

Tests/examples:

- add a `vanilla-esm` example to make sure using `import` / top-level
await works
- add a step to the `test_pkg` job to make sure vitest isn't hiding
gnarly import problems - this is installing the compiled library as a
`.tgz`, and with no other dev/prod dependencies like vitest or ts-node
having been installed, so should be very close to what end users will do

Didn't:

- add a wrapper.mjs file in the compiled folder as suggested in
#608 (comment),
mostly just because it didn't seem to be necessary? It seems to work
fine when imported from an ES-module, using top-level await, etc., even
though umzug is itself a commonjs module.

<details>
<summary>original body</summary>

~Related to #608 - although does not close it.~

~This adds built-in support for `.mjs` and `.mts` files. `.mjs` should
"just work" - write migrations as ESM modules and they'll be imported in
the right way. For the current major version, `.js` will continue to be
assumed commonjs. ESM-fans will just have to type that extra `m` in
their filenames.~

~This PR _doesn't_ add a wrapper file so that the umzug library itself
can be imported as an ES module. That can be done in a follow-up PR. In
the meantime, ESM users can use `createRequire` as in the [existing ESM
example](https://github.com/sequelize/umzug/tree/main/examples/2.es-modules).~
</details>

---------

Co-authored-by: Misha Kaletsky <[email protected]>

.github/workflows/ci.yml

@@ -54,17 +54,35 @@ jobs:
         with:
           name: tarball
       - run: ls
-      - name: install tarball in examples directory
+      - run: rm -rf examples/node_modules
+      - name: run vanilla example
         working-directory: examples/0-vanilla
         run: |
-          rm -rf ../node_modules
           npm init -y
           npm install ../../umzug.tgz
-      - name: run example
-        run: |
-          cd examples/0-vanilla
+
           node migrate up
           node migrate down
           node migrate create --name new-migration.js
           node migrate up
+      - name: run vanilla esm example
+        working-directory: examples/0.5-vanilla-esm
+        run: |
+          npm init -y
+          sed -i 's|"name"|"type": "module",\n  "name"|g' package.json
+          npm install ../../umzug.tgz
+          cat package.json
+
+          node migrate.mjs up
+          node migrate.mjs down
+          node migrate.mjs create --name new-migration-1.mjs
+          node migrate.mjs create --name new-migration-2.js
+          node migrate.mjs up
+
+          cd migrations
+          cat $(ls . | grep new-migration-1)
+          cat $(ls . | grep new-migration-2)
+
+          # hard to test this with vitest transpiling stuff for us, so make sure .mjs and .js have same content
+          cmp $(ls . | grep new-migration-1) $(ls . | grep new-migration-2)
       - run: ls -R

examples/0.5-vanilla-esm/migrate.mjs

@@ -0,0 +1,14 @@
+import { Umzug, JSONStorage } from 'umzug';
+
+const __dirname = new URL('.', import.meta.url).pathname.replace(/\/$/, '');
+
+export const migrator = new Umzug({
+	migrations: {
+		glob: 'migrations/*.*js',
+	},
+	context: { directory: __dirname + '/ignoreme' },
+	storage: new JSONStorage({ path: __dirname + '/ignoreme/storage.json' }),
+	logger: console,
+});
+
+await migrator.runAsCLI();

examples/0.5-vanilla-esm/migrations/2023.11.03T16.52.04.users-table.mjs

@@ -0,0 +1,12 @@
+import { promises as fs } from 'fs';
+
+/** @type {typeof import('../migrate.mjs').migrator['_types']['migration']} */
+export const up = async ({ context }) => {
+	await fs.mkdir(context.directory, { recursive: true });
+	await fs.writeFile(context.directory + '/users.json', JSON.stringify([], null, 2));
+};
+
+/** @type {typeof import('../migrate.mjs').migrator['_types']['migration']} */
+export const down = async ({ context }) => {
+	await fs.unlink(context.directory + '/users.json');
+};

examples/0.5-vanilla-esm/readme.md

@@ -0,0 +1,16 @@
+This example shows the simplest possible, node-only setup for Umzug. No typescript, no database, no dependencies.
+
+Note:
+- The `context` for the migrations just contains a (gitignored) directory.
+- The example migration just writes an empty file to the directory
+
+```bash
+node migrate.mjs --help # show CLI help
+
+node migrate.mjs up # apply migrations
+node migrate.mjs down # revert the last migration
+node migrate.mjs create --name new-migration.mjs # create a new migration file
+
+node migrate.mjs up # apply migrations again
+node migrate.mjs down --to 0 # revert all migrations
+```

examples/2-es-modules/umzug.mjs

@@ -1,9 +1,6 @@
-import { createRequire } from "module";
-
-const require = createRequire(import.meta.url);
-const { Umzug, SequelizeStorage } = require('umzug');
-const { Sequelize, DataTypes } = require('sequelize');
-const path = require('path');
+import { Umzug, SequelizeStorage } from 'umzug';
+import { Sequelize, DataTypes } from 'sequelize';
+import * as path from 'path';
 
 const sequelize = new Sequelize({
 	dialect: 'sqlite',
@@ -14,22 +11,6 @@ const sequelize = new Sequelize({
 export const migrator = new Umzug({
 	migrations: {
 		glob: ['migrations/*.{js,cjs,mjs}', { cwd: path.dirname(import.meta.url.replace('file://', '')) }],
-		resolve: params => {
-			if (params.path.endsWith('.mjs') || params.path.endsWith('.js')) {
-				const getModule = () => import(`file:///${params.path.replace(/\\/g, '/')}`)
-				return {
-					name: params.name,
-					path: params.path,
-					up: async upParams => (await getModule()).up(upParams),
-					down: async downParams => (await getModule()).down(downParams),
-				}
-			}
-			return {
-				name: params.name,
-				path: params.path,
-				...require(params.path),
-			}
-		}
 	},
 	context: { sequelize, DataTypes },
 	storage: new SequelizeStorage({
@@ -38,4 +19,4 @@ export const migrator = new Umzug({
 	logger: console,
 });
 
-migrator.runAsCLI()
+migrator.runAsCLI();

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "umzug",
-	"version": "3.4.0",
+	"version": "3.5.0-0",
 	"description": "Framework-agnostic migration tool for Node",
 	"keywords": [
 		"migrate",

src/umzug.ts

@@ -107,39 +107,46 @@ export class Umzug<Ctx extends object = object> extends emittery<UmzugEvents<Ctx
 		}
 
 		const ext = path.extname(filepath);
-		const canRequire = ext === '.js' || ext === '.cjs' || ext === '.ts';
 		const languageSpecificHelp: Record<string, string> = {
 			'.ts':
 				"TypeScript files can be required by adding `ts-node` as a dependency and calling `require('ts-node/register')` at the program entrypoint before running migrations.",
 			'.sql': 'Try writing a resolver which reads file content and executes it as a sql query.',
 		};
-		if (!canRequire) {
-			const errorParts = [
-				`No resolver specified for file ${filepath}.`,
-				languageSpecificHelp[ext],
-				`See docs for guidance on how to write a custom resolver.`,
-			];
-			throw new Error(errorParts.filter(Boolean).join(' '));
-		}
+		languageSpecificHelp['.cts'] = languageSpecificHelp['.ts'];
+		languageSpecificHelp['.mts'] = languageSpecificHelp['.ts'];
+
+		let loadModule: () => Promise<RunnableMigration<unknown>>;
 
-		const getModule = () => {
+		const jsExt = ext.replace(/\.([cm]?)ts$/, '.$1js');
+
+		const getModule = async () => {
 			try {
-				// eslint-disable-next-line @typescript-eslint/no-unsafe-return
-				return require(filepath);
+				return await loadModule();
 			} catch (e: unknown) {
-				if (e instanceof SyntaxError && filepath.endsWith('.ts')) {
-					e.message += '\n\n' + languageSpecificHelp['.ts'];
+				if ((e instanceof SyntaxError || e instanceof MissingResolverError) && ext in languageSpecificHelp) {
+					e.message += '\n\n' + languageSpecificHelp[ext];
 				}
 
 				throw e;
 			}
 		};
 
+		if ((jsExt === '.js' && typeof require.main === 'object') || jsExt === '.cjs') {
+			// eslint-disable-next-line @typescript-eslint/no-var-requires
+			loadModule = async () => require(filepath) as RunnableMigration<unknown>;
+		} else if (jsExt === '.js' || jsExt === '.mjs') {
+			loadModule = async () => import(filepath) as Promise<RunnableMigration<unknown>>;
+		} else {
+			loadModule = async () => {
+				throw new MissingResolverError(filepath);
+			};
+		}
+
 		return {
 			name,
 			path: filepath,
-			up: async ({ context }) => getModule().up({ path: filepath, name, context }) as unknown,
-			down: async ({ context }) => getModule().down({ path: filepath, name, context }) as unknown,
+			up: async ({ context }) => (await getModule()).up({ path: filepath, name, context }),
+			down: async ({ context }) => (await getModule()).down?.({ path: filepath, name, context }),
 		};
 	};
 
@@ -352,7 +359,7 @@ export class Umzug<Ctx extends object = object> extends emittery<UmzugEvents<Ctx
 
 			const allowedExtensions = options.allowExtension
 				? [options.allowExtension]
-				: ['.js', '.cjs', '.mjs', '.ts', '.sql'];
+				: ['.js', '.cjs', '.mjs', '.ts', '.cts', '.mts', '.sql'];
 
 			const existing = await this.migrations(context);
 			const last = existing[existing.length - 1];
@@ -415,15 +422,15 @@ export class Umzug<Ctx extends object = object> extends emittery<UmzugEvents<Ctx
 
 	private static defaultCreationTemplate(filepath: string): Array<[string, string]> {
 		const ext = path.extname(filepath);
-		if (ext === '.js' || ext === '.cjs') {
+		if ((ext === '.js' && typeof require.main === 'object') || ext === '.cjs') {
 			return [[filepath, templates.js]];
 		}
 
-		if (ext === '.ts') {
+		if (ext === '.ts' || ext === '.mts' || ext === '.cts') {
 			return [[filepath, templates.ts]];
 		}
 
-		if (ext === '.mjs') {
+		if ((ext === '.js' && typeof require.main === 'undefined') || ext === '.mjs') {
 			return [[filepath, templates.mjs]];
 		}
 
@@ -499,3 +506,9 @@ export class Umzug<Ctx extends object = object> extends emittery<UmzugEvents<Ctx
 		};
 	}
 }
+
+class MissingResolverError extends Error {
+	constructor(filepath: string) {
+		super(`No resolver specified for file ${filepath}. See docs for guidance on how to write a custom resolver.`);
+	}
+}

test/__snapshots__/examples.test.ts.snap

@@ -1,5 +1,70 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`example 0.5-vanilla-esm 1`] = `
+"\`node migrate.mjs --help\` output:
+
+...
+
+\`node migrate.mjs up\` output:
+
+{ event: 'migrating', name: '<<timestamp>>.users-table.mjs' }
+{
+  event: 'migrated',
+  name: '<<timestamp>>.users-table.mjs',
+  durationSeconds: ???
+}
+{ event: 'up', message: 'applied 1 migrations.' }
+
+\`node migrate.mjs down\` output:
+
+{ event: 'reverting', name: '<<timestamp>>.users-table.mjs' }
+{
+  event: 'reverted',
+  name: '<<timestamp>>.users-table.mjs',
+  durationSeconds: ???
+}
+{ event: 'down', message: 'reverted 1 migrations.' }
+
+\`node migrate.mjs create --name new-migration.mjs\` output:
+
+{
+  event: 'created',
+  path: '<<cwd>>/examples/0.5-vanilla-esm/migrations/<<timestamp>>.new-migration.mjs'
+}
+
+\`node migrate.mjs up\` output:
+
+{ event: 'migrating', name: '<<timestamp>>.users-table.mjs' }
+{
+  event: 'migrated',
+  name: '<<timestamp>>.users-table.mjs',
+  durationSeconds: ???
+}
+{ event: 'migrating', name: '<<timestamp>>.new-migration.mjs' }
+{
+  event: 'migrated',
+  name: '<<timestamp>>.new-migration.mjs',
+  durationSeconds: ???
+}
+{ event: 'up', message: 'applied 2 migrations.' }
+
+\`node migrate.mjs down --to 0\` output:
+
+{ event: 'reverting', name: '<<timestamp>>.new-migration.mjs' }
+{
+  event: 'reverted',
+  name: '<<timestamp>>.new-migration.mjs',
+  durationSeconds: ???
+}
+{ event: 'reverting', name: '<<timestamp>>.users-table.mjs' }
+{
+  event: 'reverted',
+  name: '<<timestamp>>.users-table.mjs',
+  durationSeconds: ???
+}
+{ event: 'down', message: 'reverted 2 migrations.' }"
+`;
+
 exports[`example 0-vanilla 1`] = `
 "\`node migrate --help\` output:
 

test/cli.test.ts

@@ -244,15 +244,15 @@ describe('create migration file', () => {
 		// a folder must be specified for the first migration
 		await expect(runCLI(['create', '--name', 'm1.js', '--folder', path.join(syncer.baseDir, 'migrations')])).resolves
 			.toMatchInlineSnapshot(`
-		{
-		  "2000.01.02T00.00.00.m1.js": "/** @type {import('umzug').MigrationFn<any>} */
-		exports.up = async params => {};
+			{
+			  "2000.01.02T00.00.00.m1.js": "/** @type {import('umzug').MigrationFn<any>} */
+			export const up = async params => {};
 
-		/** @type {import('umzug').MigrationFn<any>} */
-		exports.down = async params => {};
-		",
-		}
-	`);
+			/** @type {import('umzug').MigrationFn<any>} */
+			export const down = async params => {};
+			",
+			}
+		`);
 
 		// for the second migration, the program should guess it's supposed to live next to the previous one.
 		await expect(runCLI(['create', '--name', 'm2.ts'])).resolves.toMatchInlineSnapshot(`
@@ -278,7 +278,7 @@ describe('create migration file', () => {
 	`);
 
 		await expect(runCLI(['create', '--name', 'm4.txt'])).rejects.toThrowErrorMatchingInlineSnapshot(
-			`"Extension .txt not allowed. Allowed extensions are .js, .cjs, .mjs, .ts, .sql. See help for --allow-extension to avoid this error."`
+			'"Extension .txt not allowed. Allowed extensions are .js, .cjs, .mjs, .ts, .cts, .mts, .sql. See help for --allow-extension to avoid this error."'
 		);
 
 		await expect(runCLI(['create', '--name', 'm4.txt', '--allow-extension', '.txt'])).rejects.toThrow(

test/examples.test.ts

@@ -2,7 +2,11 @@ import * as fs from 'fs';
 import * as path from 'path';
 import stripAnsi from 'strip-ansi';
 import execa from 'execa';
-import { test, expect } from 'vitest';
+import { test, expect, beforeAll } from 'vitest';
+
+beforeAll(async () => {
+	await execa('npm', ['run', 'compile']);
+});
 
 const examplesDir = path.join(__dirname, '../examples');
 const examples = fs.readdirSync(examplesDir).filter(ex => /^\d/.exec(ex));