build: hoist entire wasm build into this project

Including both talvos and its dependencies. Now, we ought to be able to
produce the entire tool from a single (submodule-recursive) clone of
this project alone, without requiring external setup/pre-build steps.

Also, prior to this snapshot, invoking the build-wasm script in watch
mode wouldn't watch for changes when the initial first-pass build
failed. Now, it will watch a failing build, at the cost of moving the
main flow "out of line."

.dockerignore

@@ -1,3 +1,5 @@
 node_modules
 .git
 dist
+wasm/build
+tools/target

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "wasm/talvos"]
 	path = wasm/talvos
 	url = https://github.com/sethp/talvos/
+[submodule "wasm/SPIRV-Headers"]
+	path = wasm/SPIRV-Headers
+	url = https://github.com/KhronosGroup/SPIRV-Headers.git
+[submodule "wasm/SPIRV-Tools"]
+	path = wasm/SPIRV-Tools
+	url = https://github.com/KhronosGroup/SPIRV-Tools.git

.vscode/settings.json

@@ -21,14 +21,9 @@
     "--enable-config",
     "--background-index",
     "-j=16",
-    "--compile-commands-dir=${workspaceFolder}/wasm/talvos/build"
+    "--compile-commands-dir=${workspaceFolder}/wasm/build"
   ],
-  "cmake.buildDirectory": "${workspaceFolder}/wasm/talvos/build",
-  "cmake.sourceDirectory": "${workspaceFolder}/wasm/talvos",
-  "cmake.configureArgs": [
-    "-DSPIRV_INCLUDE_DIR=${workspaceFolder}/tmp/_SPIRV-Headers/include",
-    "-DSPIRV_TOOLS_INCLUDE_DIR=${workspaceFolder}/tmp/_SPIRV-Tools/include",
-    "-DSPIRV_TOOLS_LIBRARY_DIR=${workspaceFolder}/tmp/_SPIRV-Tools/build/enscripten/source"
-  ],
-  "iwyu.compile_commands": "${workspaceFolder}/wasm/talvos/build/compile_commands.json"
+  "cmake.buildDirectory": "${workspaceFolder}/wasm/build",
+  "cmake.sourceDirectory": "${workspaceFolder}/wasm/",
+  "iwyu.compile_commands": "${workspaceFolder}/wasm/build/compile_commands.json"
 }

Dockerfile.build

@@ -19,7 +19,6 @@ set -euxo pipefail
 
 declare -rA LINKS=(
 	["../styles"]="/"
-	["/tmp/talvos"]="/@fs/home/seth/Code/src/github.com/sethp/learn-gpgpu/tmp/_talvos"
 )
 
 mkdir -p public
@@ -33,10 +32,6 @@ for TARGET in ${!LINKS[@]}; do
 done
 EOS
 
-#  http://localhost:3000/@fs/home/seth/Code/src/github.com/sethp/learn-gpgpu/tmp/_talvos/lib/talvos/Block.cpp
-
-# TODO[seth]: ./tmp/_talvos_cpy is sourced from
-# `cp -r tmp/_talvos/{include,lib,tools/talvos-cmd} tmp/_talvos_cpy/`
 RUN \
 	--mount=type=bind,source=astro,target=/app/astro \
 	--mount=type=bind,source=content,target=/app/content \
@@ -46,7 +41,6 @@ RUN \
 	--mount=type=bind,source=wasm,target=/app/wasm \
 	--mount=type=bind,source=./astro.config.ts,target=/app/astro.config.ts \
 	--mount=type=bind,source=./tsconfig.json,target=/app/tsconfig.json \
-	--mount=type=bind,source=./tmp/_talvos_cpy,target=/tmp/talvos \
 	pnpm run astro build
 
 # Post-fixups

NOTES.md

@@ -21,8 +21,13 @@ git -C wasm/talvos commit -av
 git add wasm/talvos
 ```
 
-Will `git push` push both? (probably not)
+Will `git push` push both? No, confirmed it does not.
 
+Instead:
+```
+git -C wasm/talvos push
+git push
+```
 
 # entr
 
@@ -199,3 +204,105 @@ $ cmake --install build/enscripten # sic
 ```
 
 ah, and now I get a compile_commands.json and "all" is "well".
+
+# cmake "superproject" build
+
+Why do a single CMake "superproject"? The IDE tooling all wants a single `compile_commands.json`, and this seemed the most expedient way to glom them all together. The sticky part was getting the ad-hoc discovery mechanism(s) to all agree. The library part was especially strange, since it has to be a `SPIRV-Tools` magic string (that matches the target name) in "superproject" mode for CMake to understand the dependency link, but then `wasm-ld` wasn't able to find it in subproject mode. So, one more layer of indirection!
+
+Talvos was doing discovery differently than everyone else, and I couldn't figure out how to get the `find_library` call to work at all with discovering a the SPIRV-Tools as a sibling target. Luckily, (ab)using `find_project` made for a CMakeLists.txt that worked for both talvos stand-alone (with system-installed headers/tools), and also as part of the combined cmake project.
+
+Moving the build "up" a level was pretty straightforward after that:
+
+```
+   cp --verbose --update=older build/emscripten-docker/tools/talvos-cmd/talvos-wasm.* "$1"
+-> cp --verbose --update=older build/emscripten-docker/talvos/tools/talvos-cmd/talvos-wasm.* "$1"
+```
+
+(as an aside: cmake's namespacing doesn't make a lot of sense to me. Every `add_subdirectory` gets reflected in the build output, but the `target` namespace is global, at least by default?)
+
+## avoiding double-building SPIRV-Tools
+
+Is it sensible to try and have a layer-per-dependency in the docker build? I'm not sure, but I had it before I moved over to submodules for the two dependencies.
+
+This ought to be possible by preferring that `find_package` locates the already-built SPIRV-Tools instead of the sibling project. There seems to be a mode suited for this task:
+
+  https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_PACKAGE_PREFER_CONFIG.html
+
+But it doesn't work on its own (it's still rebuilding SPIRV-Tools, even though libSPIRV-Tools.a is already in the sysroot). Maybe it's finding the project via https://cmake.org/cmake/help/latest/command/find_package.html#id9 ("Config Module Search Procedure") ?
+
+Once `-USPIRV-Tools_DIR` was being passed to the configure step (thanks, cache vars!), then I tried:
+
+```
+find_package(SPIRV-Tools REQUIRED NO_MODULE
+             NO_DEFAULT_PATH
+             NO_PACKAGE_ROOT_PATH
+             NO_CMAKE_PATH
+             NO_CMAKE_ENVIRONMENT_PATH
+             NO_SYSTEM_ENVIRONMENT_PATH
+             NO_CMAKE_PACKAGE_REGISTRY
+             NO_CMAKE_BUILDS_PATH
+             NO_CMAKE_SYSTEM_PATH # vs. NO_CMAKE_INSTALL_PREFIX
+             NO_CMAKE_SYSTEM_PACKAGE_REGISTRY
+             NO_CMAKE_FIND_ROOT_PATH
+)
+
+if (NOT SPIRV-Tools_DIR)
+	message(FATAL_ERROR "no SPIRV-Tools found")
+else()
+	message(STATUS "SPIRV-Tools found ${SPIRV-Tools_DIR}")
+endif()
+```
+
+which successfully failed (I've turned off all of the possible search locations). That let me bisect:
+
+```
+# find_package(SPIRV-Tools REQUIRED NO_MODULE)
+# -> found /emsdk/upstream/emscripten/cache/sysroot/lib/cmake/SPIRV-Tools
+
+# find_package(SPIRV-Tools REQUIRED NO_MODULE
+#              NO_DEFAULT_PATH
+#              NO_PACKAGE_ROOT_PATH
+#              NO_CMAKE_PATH
+#              NO_CMAKE_ENVIRONMENT_PATH
+#              NO_SYSTEM_ENVIRONMENT_PATH
+#              NO_CMAKE_PACKAGE_REGISTRY
+#              NO_CMAKE_BUILDS_PATH
+#              NO_CMAKE_SYSTEM_PATH # vs. NO_CMAKE_INSTALL_PREFIX
+#              NO_CMAKE_SYSTEM_PACKAGE_REGISTRY
+#              NO_CMAKE_FIND_ROOT_PATH
+# )
+# -> not found (expected)
+
+# find_package(SPIRV-Tools REQUIRED NO_MODULE
+#              NO_CMAKE_PACKAGE_REGISTRY
+#              NO_CMAKE_BUILDS_PATH
+#              NO_CMAKE_SYSTEM_PATH # vs. NO_CMAKE_INSTALL_PREFIX
+#              NO_CMAKE_SYSTEM_PACKAGE_REGISTRY
+#              NO_CMAKE_FIND_ROOT_PATH
+# )
+# -> not found (so, the "finder" ought to be in the other half)
+
+find_package(SPIRV-Tools REQUIRED NO_MODULE
+             NO_DEFAULT_PATH
+             NO_PACKAGE_ROOT_PATH
+             NO_CMAKE_PATH
+             NO_CMAKE_ENVIRONMENT_PATH
+             NO_SYSTEM_ENVIRONMENT_PATH
+)
+# -> not found (?!)
+```
+
+which immediately revealed there's something wacky going on in there (probably two of the options overlap, there's 10 of them and only 9 "steps" on the documentation page). But also, that neither mechanism is doing what I want because this is what they're ending up resolving to:
+
+```
+# cat /emsdk/upstream/emscripten/cache/sysroot/lib/cmake/SPIRV-Tools/SPIRV-ToolsConfig.cmake
+include(${CMAKE_CURRENT_LIST_DIR}/SPIRV-ToolsTarget.cmake)
+if(TARGET SPIRV-Tools)
+    set(SPIRV-Tools_LIBRARIES SPIRV-Tools)
+    get_target_property(SPIRV-Tools_INCLUDE_DIRS SPIRV-Tools INTERFACE_INCLUDE_DIRECTORIES)
+endif()
+```
+
+`if(TARGET SPIRV-Tools)` it just goes ahead and says "yeah, link against the sibling project": the opposite of my goal. Cool.
+
+I had been trying to avoid inventing a project-specific convention for "use find_library instead of find_project", oh well.

astro.config.ts

@@ -19,12 +19,14 @@ export default defineConfig({
 	vite: {
 		server: {
 			watch: {
+				// see also: `--mount` list in Dockerfile.build
 				ignored: [
+					'**/__snapshots__/**',
+					'**/dist/**',
+					'**/hack/**',
 					'**/tmp/**',
-					'**/wasm/talvos/**',
 					'**/tools/**',
-					'**/dist/**',
-					'**/__snapshots__/**',
+					'**/wasm/*/**',
 				],
 			},
 		},

dist/index.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html><html><head><title>gpgpu playground</title><link rel="stylesheet" type="text/css" media="all" href="/learn-gpgpu/styles/index.css"><script type="module" src="/learn-gpgpu/_astro/hoisted.e17a5b3a.js"></script></head><body><div id="talvos"><div><main><textarea class="module" autocorrect="off" autocomplete="off" autocapitalize="off" spellcheck="false" aria-label="SPIR-V module assembly (spv_text format)" wrap="off">; SPIR-V
+<!DOCTYPE html><html><head><title>gpgpu playground</title><link rel="stylesheet" type="text/css" media="all" href="/learn-gpgpu/styles/index.css"><script type="module" src="/learn-gpgpu/_astro/hoisted.520093a8.js"></script></head><body><div id="talvos"><div><main><textarea class="module" autocorrect="off" autocomplete="off" autocapitalize="off" spellcheck="false" aria-label="SPIR-V module assembly (spv_text format)" wrap="off">; SPIR-V
 ; Version: 1.2
 ; Generator: Khronos SPIR-V Tools Assembler; 0
 ; Bound: 32

hack/build-wasm.sh

@@ -46,9 +46,11 @@ done
 
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 
-./wasm/talvos/hack/build-wasm.sh "$PWD/wasm"
+[[ -v WATCH_MODE ]] || {
+	./hack/build/talvos-wasm.sh "$PWD/wasm"
 
-[[ -v WATCH_MODE ]] || exit 0;
+	exit 0;
+}
 
 watch() {
 	# via https://superuser.com/a/665208
@@ -57,14 +59,17 @@ watch() {
 
 		(
 			(
-				echo wasm/talvos/Dockerfile.emscripten
+				find hack/build -type f
+				echo wasm/CMakeLists.txt
+				echo wasm/Dockerfile.emscripten
+				find wasm/cmake -type f
 				find wasm/talvos -name 'build' -prune -o \( -name 'CMakeLists.txt' -o -name '*.cmake' \) -print
 				find wasm/talvos -name 'build' -prune -o -name '*.h' -print
 				find wasm/talvos -name '*.c' -o -name '*.cpp'
 			) \
 		| tee ${VERBOSE+/dev/stderr} \
 		) \
-		| entr -c -d -p "$0" ${VERBOSE+-v} # no --watch
+		| entr -c -d "$0" ${VERBOSE+-v} # no --watch
 
 		# via `man entr`
 		#  2       A file was added to a directory and the directory watch option was specified
@@ -73,6 +78,8 @@ watch() {
 		[[ -v VERBOSE ]] && echo >&2 "Restarting watcher";
 	done
 
+	echo "All done! Exiting..."
+
 	# important to keep bash from executing "off the end" of the file if it's changed
 	exit
 }

hack/build/talvos-wasm.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+set -euo pipefail
+
+[[ $# -gt 0 ]] || {
+    echo >&2 "usage: $0 PATH"
+    echo >&2 "missing required PATH to copy artifacts into once they're built"
+    echo >&2 "e.g. $0 ../learn-gpgpu/wasm"
+    exit 2
+}
+
+uniqifier() {
+	# we use the inode of the tmpfile as part of the uniqifier
+	# `mktemp` effectively reserves a name for our use, but
+	# leaves a dangling file around unless we clean it up. using
+	# the inode lets us do the "unlink an open file" trick, while
+	# still reserving a unique name.
+	#
+	# see also: https://unix.stackexchange.com/a/181938
+	#
+	# why do this tortured, cursed thing? it handles:
+	# * concurrent runs of this script[0]
+	# * unexpected exits (even `kill -9`, after the rm)
+	# * temp file "fixation" races
+	#
+	# remove when: docker provides a way to get the unique content sha
+	# back out of a `docker build` other than `-q` which hides the output.
+	#
+	# [0] or indeed any script using the same docker repo prefix,
+	#     as long as they share the same tmp directory or filesystem
+	! [[ -e /proc/$$/fd/45 ]] || {
+		# since bash 4.1 there's also a way to get the shell to allocate a fd for us,
+		# but the default bash on _some_ platforms (macOS) is still version 3.
+		# cf. https://stackoverflow.com/a/41620630
+		echo >&2 "cannot generate uniqifier: file descriptor 45 already in use; did you call it twice?"
+		echo >&2 "(if you need two unique tags, try making a \`uniqifier2\` copy and changing the file descriptor number)"
+		exit 1
+	}
+	tmpfile=$(mktemp)
+	exec 45<"$tmpfile"
+	inode=$(stat -c'%i' "$tmpfile")
+	rm "$tmpfile"
+
+	echo "$(basename "$tmpfile")-ino-$inode"
+}
+TAG="talvos-build:$(uniqifier)"
+
+# best effort; it'll be a while before the docker daemon will delete the layers anyway
+# TODO: this is a little too eager, most of the time (even with --no-prune). I'd like
+#       the image to stick around in case I want to get in and poke at something
+# cleanup_tag() {
+# 	docker image rm --no-prune "$TAG" || true
+# }
+# trap cleanup_tag EXIT
+
+set -x
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../../wasm"
+
+# ensure we make the `./build` directory outside of the container, so
+# it's owned by whoever's running this script
+mkdir -p ./build/
+
+# TODO switch this over to a -o type=local,dest=... kind of build
+#      so we can avoid all these tag/multiple `docker run`s / stateful cmake build dir shenanigans
+docker build -f Dockerfile.emscripten . --target=talvos -t "$TAG"
+
+# [ -d ./build/emscripten-docker ] || {
+# 	docker run -it --rm -v "$(pwd)":/usr/src -w /usr/src/talvos \
+#     "$TAG" -- \
+#     cmake -B build/emscripten-docker
+# }
+
+# these flags clear the "found" part of `find_packages`'s config mode (and maybe module mode?), so resolution
+# is re-run.
+# see: https://cmake.org/cmake/help/latest/command/find_package.html
+# > Config mode search attempts to locate a configuration file provided by the package to be found.
+# > A cache entry called <PackageName>_DIR is created to hold the directory containing the file.
+CMAKE_UNSET_FOUND_PACKAGE_CACHE_VARS=(
+	-USPIRV-Header_DIR
+	-USPIRV-Tools_DIR
+)
+
+docker run -it --rm -v "$(pwd)":/usr/src -w /usr/src \
+	"$TAG" -- \
+	cmake -B build/emscripten-docker  \
+	  -DCMAKE_C_FLAGS="-fdebug-compilation-dir=. -ffile-prefix-map=/usr/src=../wasm" \
+    -DCMAKE_CXX_FLAGS="-fdebug-compilation-dir=. -ffile-prefix-map=/usr/src=../wasm" \
+		"${CMAKE_UNSET_FOUND_PACKAGE_CACHE_VARS[@]}" \
+		-DUSE_INSTALLED_SPIRV-Tools:BOOL=TRUE \
+		-DCMAKE_FIND_PACKAGE_PREFER_CONFIG:BOOL=TRUE # <-- doesn't seem to work in terms of avoiding rebuilding SPIRV-Tools twice
+
+
+# TODO something about this?
+# ```
+# cache:INFO: generating system asset: symbol_lists/1701111f65fe30b91927855eb0451158a36e4416.json... (this will be cached in "/emsdk/upstream/emscripten/cache/symbol_lists/1701111f65fe30b91927855eb0451158a36e4416.json" for subsequent builds)
+# ```
+
+docker run -it --rm -v "$(pwd)":/usr/src -w /usr/src \
+    "$TAG" -- \
+    cmake --build build/emscripten-docker --target talvos-wasm
+
+cp --verbose --update=older build/emscripten-docker/talvos/tools/talvos-cmd/talvos-wasm.* "$1"

wasm/.gitignore

@@ -0,0 +1 @@
+/build

wasm/CMakeLists.txt

@@ -0,0 +1,10 @@
+
+cmake_minimum_required(VERSION 3.17.2)
+
+project(wasm)
+
+set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules)
+
+add_subdirectory(SPIRV-Headers)
+add_subdirectory(SPIRV-Tools)
+add_subdirectory(talvos)