feat: typescript and esm in preflight (#6157)

Fixes #3013

~TODO~
- [x] Check benchmarks to make sure happy path perf isn't hurt
- [x] Windows
- [x] ~Add e2e test to verify fix for `"type": "module"`~ Doesn't fix that issue anymore
- [x] Docs to officially sanction using ts in both preflight and inflight via extern

*By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*.

apps/wingcli-v2/src/main.rs

@@ -10,8 +10,6 @@ use lazy_static::lazy_static;
 use strum::{Display, EnumString};
 use wingc::compile;
 
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-
 lazy_static! {
 	static ref HOME_PATH: PathBuf = home_dir().expect("Could not find home directory");
 	pub static ref WING_CACHE_DIR: Utf8PathBuf =
@@ -90,6 +88,10 @@ fn command_build(source_file: Utf8PathBuf, target: Option<Target>) -> Result<(),
 		install_sdk()?;
 	} else {
 		// TODO: check that the SDK version matches the CLI version
+		if cfg!(test) {
+			// For now, always reinstall the SDK in tests
+			install_sdk()?;
+		}
 	}
 	tracing::info!("Using SDK at {}", sdk_root);
 
@@ -117,10 +119,13 @@ fn install_sdk() -> Result<(), Box<dyn Error>> {
 
 	std::fs::create_dir_all(WING_CACHE_DIR.as_str())?;
 	let mut install_command = std::process::Command::new("npm");
-	install_command
-		.arg("install")
-		.arg(format!("@winglang/sdk@{VERSION}"))
-		.arg("esbuild"); // TODO: should this not be an optional dependency?
+	install_command.arg("install").arg("esbuild"); // TODO: should this not be an optional dependency?
+	if cfg!(test) {
+		install_command.arg(format!("file:{}/../../libs/wingsdk", env!("CARGO_MANIFEST_DIR")));
+	} else {
+		install_command.arg(format!("@winglang/sdk@{}", env!("CARGO_PKG_VERSION")));
+	}
+
 	install_command.current_dir(WING_CACHE_DIR.as_str());
 	install_command.stdout(std::process::Stdio::piped());
 	install_command.stderr(std::process::Stdio::piped());

docs/docs/03-language-reference.md

@@ -1885,7 +1885,8 @@ Mapping JSII types to Wing types:
 
 ## 5.2 JavaScript
 
-The `extern "<commonjs module path>"` modifier can be used on method declarations in classes to indicate that a method is backed by an implementation imported from a JavaScript module. The module must be a relative path and will be loaded via [require()](https://nodejs.org/api/modules.html#requireid).
+The `extern "<javascript module path>"` modifier can be used on method declarations in classes to indicate that a method is backed by an implementation imported from a JavaScript module. The module must be a relative path and will be loaded via [require()](https://nodejs.org/api/modules.html#requireid).
+This module can be either CJS or ESM and may be written in JavaScript or TypeScript.
 
 In the following example, the static inflight method `makeId` is implemented
 in `helper.js`:
@@ -1920,13 +1921,7 @@ matching name (without any case conversion).
 
 Extern methods do not support access to class's members through `this`, so they must be declared `static`.
 
-### 5.2.1 TypeScript
-
-It is possible to use TypeScript to write helpers, but at the moment this is not
-directly supported by Wing. This means that you will need to setup the TypeScript toolchain
-to compile your code to JavaScript and then use `extern` against the JavaScript file.
-
-### 5.2.2 Type model
+### 5.2.1 Type model
 
 The table below shows the mapping between Wing types and JavaScript values, shown with TypeScript types.
 When calling **extern** function, the parameter and return types are **assumed** to be satisfied by the called function.

docs/docs/07-examples/10-using-javascript.md

@@ -6,7 +6,7 @@ keywords: [example, javascript, extern, typescript, js, ts]
 
 Calling a Javascript function from Wing requires two steps.
 
-1. Create a .js file that exports some functions
+1. Create a .js/.ts file that exports some functions
 
 ```js
 // util.js
@@ -16,7 +16,7 @@ exports.isValidUrl = function (url) {
 };
 ```
 
-In preflight, this file must be a CommonJS module written in Javascript. Inflight, it may be CJS/ESM and either JavaScript or TypeScript.
+It may be CJS/ESM written in either JavaScript or TypeScript.
 
 2. Use the `extern` keyword in a class to expose the function to Wing. Note that this must be `static`. It may also be `inflight`
 

examples/tests/invalid/extern_static.test.w

@@ -1,4 +1,4 @@
 class Foo {
-  extern "../valid/external_js.js" inflight getGreeting(name: str): str;
+  extern "../valid/external_ts.ts" inflight getGreeting(name: str): str;
                                     //^ Error: extern methods must be declared "static"
 }

examples/tests/invalid/lib/extern_above.w

@@ -1,4 +1,4 @@
 class Foo1 {
-  extern "../../valid/external_js.js" static getGreeting(name: str): str;
+  extern "../../valid/external_ts.ts" static getGreeting(name: str): str;
        //^ must be a sub directory of the entrypoint
 }

examples/tests/valid/bring_local.test.w

@@ -5,15 +5,21 @@ bring "./store.w" as file1;
 bring "./subdir/subfile.w" as file2;
 bring "./subdir/empty.w" as file3;
 bring math;
+bring expect;
 
 // classes from other files can be used
 let store = new file1.Store();
 let q = new file2.Q();
+expect.equal(file2.Q.preflightGreet("foo"), "Hello foo");
 
 test "add data to store" {
   store.store("foo");
 }
 
+test "greet" {
+  expect.equal(file2.Q.greet("bar"), "Hello bar");
+}
+
 // structs from other files can be used
 let s = file1.Point {
   x: 1,

examples/tests/valid/extern_implementation.test.w

@@ -1,12 +1,12 @@
 bring cloud;
 
 class Foo {
-  extern "./external_js.js" pub static getGreeting(name: str): str;
-  extern "./external_js.js" static inflight regexInflight(pattern: str, text: str): bool;
-  extern "./external_js.js" static inflight getUuid(): str;
-  extern "./external_js.js" static inflight getData(): str;
-  extern "./external_js.js" pub static inflight print(msg: str): void;
-  extern "./external_js.js" pub static preflightBucket(bucket: cloud.Bucket, id: str): Json;
+  extern "./external_ts.ts" pub static getGreeting(name: str): str;
+  extern "./external_ts.ts" static inflight regexInflight(pattern: str, text: str): bool;
+  extern "./external_ts.ts" static inflight getUuid(): str;
+  extern "./external_ts.ts" static inflight getData(): str;
+  extern "./external_ts.ts" pub static inflight print(msg: str): void;
+  extern "./external_ts.ts" pub static preflightBucket(bucket: cloud.Bucket, id: str): void;
 
   pub inflight call() {
     assert(Foo.regexInflight("[a-z]+-\\d+", "abc-123"));
@@ -22,10 +22,14 @@ assert(Foo.getGreeting("Wingding") == "Hello, Wingding!");
 let f = new Foo();
 
 let bucket = new cloud.Bucket() as "my-bucket";
-let result = Foo.preflightBucket(bucket, "my-bucket");
+Foo.preflightBucket(bucket, "my-bucket");
 
-test "call" {
+let func = new cloud.Function(inflight () => {
   f.call();
+});
+
+test "call" {
+  func.invoke();
 }
 
 test "console" {

examples/tests/valid/external_js.extern.d.ts → examples/tests/valid/external_ts.extern.d.ts

@@ -2,7 +2,7 @@ export default interface extern {
   getData: () => Promise<string>,
   getGreeting: (name: string) => string,
   getUuid: () => Promise<string>,
-  preflightBucket: (bucket: Bucket, id: string) => Readonly<any>,
+  preflightBucket: (bucket: Bucket, id: string) => void,
   print: (msg: string) => Promise<void>,
   regexInflight: (pattern: string, text: string) => Promise<boolean>,
 }

examples/tests/valid/external_ts.ts

@@ -0,0 +1,31 @@
+import assert from "assert";
+import extern from "./external_ts.extern";
+
+export const getGreeting: extern["getGreeting"] = function (name) {
+  return `Hello, ${name}!`;
+};
+
+export const regexInflight: extern["regexInflight"] = async function (
+  pattern,
+  text
+) {
+  const regex = new RegExp(pattern);
+  return regex.test(text);
+};
+
+export const getUuid: extern["getUuid"] = async function () {
+  let uuid = require("uuid");
+  return uuid.v4();
+};
+
+export const getData: extern["getData"] = async function () {
+  return require("./exported_data.js");
+};
+
+export const print: extern["print"] = async function (msg) {
+  console.log(`printing ${msg}`);
+};
+
+export const preflightBucket: extern["preflightBucket"] = (bucket, id) => {
+  assert.strictEqual(bucket.node.id, id);
+};

examples/tests/valid/subdir/subfile.w

@@ -1,5 +1,6 @@
 bring math;
 
 pub class Q {
-  extern "./util.ts" static inflight greet(name: str): str;
+  pub extern "./util.ts" static inflight greet(name: str): str;
+  pub extern "./util.ts" static preflightGreet(name: str): str;
 }

examples/tests/valid/subdir/util.extern.d.ts

@@ -1,3 +1,4 @@
 export default interface extern {
   greet: (name: string) => Promise<string>,
+  preflightGreet: (name: string) => string,
 }

examples/tests/valid/subdir/util.ts

@@ -2,4 +2,8 @@ import type extern from "./util.extern";
 
 export const greet: extern["greet"] = async (name) => {
   return "Hello " + name;
+}
+
+export const preflightGreet: extern["preflightGreet"] = (name) => {
+  return "Hello " + name;
 } 

libs/wingc/src/dtsify/snapshots/declarations.snap

@@ -79,6 +79,7 @@ module.exports = function({  }) {
 const $stdlib = require('@winglang/sdk');
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 module.exports = {
   ...require("./preflight.lib-1.cjs"),
 };
@@ -98,6 +99,7 @@ export * from "./preflight.lib-1.cjs"
 const $stdlib = require('@winglang/sdk');
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class ParentClass extends $stdlib.std.Resource {
   constructor($scope, $id, ) {
     super($scope, $id);

libs/wingc/src/dtsify/snapshots/optionals.snap

@@ -45,6 +45,7 @@ module.exports = function({  }) {
 const $stdlib = require('@winglang/sdk');
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 module.exports = {
   ...require("./preflight.lib-1.cjs"),
 };
@@ -64,6 +65,7 @@ export * from "./preflight.lib-1.cjs"
 const $stdlib = require('@winglang/sdk');
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class ParentClass extends $stdlib.std.Resource {
   constructor($scope, $id, ) {
     super($scope, $id);

libs/wingc/src/jsify.rs

@@ -48,6 +48,7 @@ const ENV_WING_IS_TEST: &str = "$wing_is_test";
 const OUTDIR_VAR: &str = "$outdir";
 const PLATFORMS_VAR: &str = "$platforms";
 const HELPERS_VAR: &str = "$helpers";
+const EXTERN_VAR: &str = "$extern";
 
 const ROOT_CLASS: &str = "$Root";
 const JS_CONSTRUCTOR: &str = "constructor";
@@ -170,7 +171,7 @@ impl<'a> JSifier<'a> {
 
 		output.line("\"use strict\";");
 
-		output.line(format!("const {} = require('{}');", STDLIB, STDLIB_MODULE));
+		output.line(format!("const {STDLIB} = require('{STDLIB_MODULE}');"));
 
 		if is_entrypoint {
 			output.line(format!(
@@ -187,6 +188,9 @@ impl<'a> JSifier<'a> {
 		// "std" is implicitly imported
 		output.line(format!("const std = {STDLIB}.{WINGSDK_STD_MODULE};"));
 		output.line(format!("const {HELPERS_VAR} = {STDLIB}.helpers;"));
+		output.line(format!(
+			"const {EXTERN_VAR} = {HELPERS_VAR}.createExternRequire(__dirname);"
+		));
 		output.add_code(imports);
 
 		if is_entrypoint {
@@ -1452,9 +1456,15 @@ impl<'a> JSifier<'a> {
 						format!("{up_dirs}{rel_path}")
 					};
 
+				let require = if ctx.visit_ctx.current_phase() == Phase::Inflight {
+					"require"
+				} else {
+					EXTERN_VAR
+				};
+
 				new_code!(
 					&func_def.span,
-					format!("return (require(\"{require_path}\")[\"{name}\"])("),
+					format!("return ({require}(\"{require_path}\")[\"{name}\"])("),
 					parameters.clone(),
 					")"
 				)

libs/wingc/src/jsify/snapshots/access_methods_and_properties_on_collections.snap

@@ -47,6 +47,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);

libs/wingc/src/jsify/snapshots/access_property_on_primitive.snap

@@ -44,6 +44,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);

libs/wingc/src/jsify/snapshots/access_property_on_value_returned_from_collection.snap

@@ -44,6 +44,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);

libs/wingc/src/jsify/snapshots/base_class_captures_inflight.snap

@@ -68,6 +68,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);

libs/wingc/src/jsify/snapshots/base_class_captures_preflight.snap

@@ -62,6 +62,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);

libs/wingc/src/jsify/snapshots/base_class_lift_indirect.snap

@@ -76,6 +76,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 const cloud = $stdlib.cloud;
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {

libs/wingc/src/jsify/snapshots/base_class_with_fields_inflight.snap

@@ -77,6 +77,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);

libs/wingc/src/jsify/snapshots/base_class_with_fields_preflight.snap

@@ -67,6 +67,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);

libs/wingc/src/jsify/snapshots/base_class_with_lifted_field_object.snap

@@ -67,6 +67,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 const cloud = $stdlib.cloud;
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {

libs/wingc/src/jsify/snapshots/base_class_with_lifted_fields.snap

@@ -67,6 +67,7 @@ const $outdir = process.env.WING_SYNTH_DIR ?? ".";
 const $wing_is_test = process.env.WING_IS_TEST === "true";
 const std = $stdlib.std;
 const $helpers = $stdlib.helpers;
+const $extern = $helpers.createExternRequire(__dirname);
 class $Root extends $stdlib.std.Resource {
   constructor($scope, $id) {
     super($scope, $id);