Merge (#197)

* gh-action: ossf/scorecard-action#997

* dockerfile: node 22

* gh-action: wrangler 3.56

* wrangler: upload source maps

* fly: mmap to read trie on disk

* node: rmv webpack for backend

* use mmap for node 22

* node: type assertions with type accessors

* node: use @aryaskov/mmap-io

* deno: v1.44.4

* node: fix container cwd

* gh-action: denoland/deployctl version

* deno: import_map mmap-io

* fly: webpack bundle target node 22

* fly: webpack externalize native module mmap

* node: rmv unused var from blocklists.js

* node: omit dev deps in docker

* fly: bundle node_modules instead of copying

brings down image size down from 900+mb (400mb+ due to node_modules, mostly
devDependencies) to 300mb+

* fly: do not omit-dev in setup (dep: webpack)

* fly: fix entrypoint

* gh-action: ghcr for node-alpine mk1

---------

Co-authored-by: ignoramous <[email protected]>
Co-authored-by: Murtaza Aliakbar <[email protected]>

.github/workflows/cf.yml

@@ -52,6 +52,7 @@ on:
 
 env:
   GIT_REF: ${{ github.event.inputs.commit || github.ref }}
+  WRANGLER_VER: '3.56.0'
   # default is 'dev' which is really empty/no env
   WORKERS_ENV: ''
 
@@ -118,6 +119,7 @@ jobs:
           accountId: ${{ secrets.CF_ACCOUNT_ID }}
           # input overrides env-defaults, regardless
           environment: ${{ env.WORKERS_ENV }}
+          wranglerVersion: ${{ env.WRANGLER_VER }}
         env:
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.NEXTDNS_CONF }}
           GIT_COMMIT_ID: ${{ env.COMMIT_SHA }}

.github/workflows/deno-deploy.yml

@@ -96,7 +96,7 @@ jobs:
       - name: 🦕 Install Deno @1.29
         uses: denoland/setup-deno@5fae568d37c3b73449009674875529a984555dd1 # main
         with:
-          deno-version: 1.29.3
+          deno-version: 1.44.4
 
       - name: 📦 Bundle up
         if: ${{ env.DEPLOY_MODE == 'action' }}

.github/workflows/ghcr.yml

@@ -0,0 +1,60 @@
+name: 🔄 runc
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+env:
+  REGISTRY: "ghcr.io"
+  IMAGE_NAME: ${{ github.repository }}
+  GIT_REF: ${{ github.event.inputs.git-ref || github.ref }}
+
+# docs.github.com/en/actions/publishing-packages/publishing-docker-images
+jobs:
+  nodejs:
+    name: 🚀 Node on Alpine
+    runs-on: ubuntu-latest
+    permissions:
+        contents: read
+        packages: write
+        attestations: write
+        id-token: write
+
+    steps:
+      - name: 🚚 Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+          fetch-depth: 0
+
+      - name: 🔐 Login
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🏷️ Metadata
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: 🛠 Build 
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: ./node.Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: 📕 Attest
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.github/workflows/scorecard.yml

@@ -37,12 +37,20 @@ jobs:
           egress-policy: audit
 
       - name: "Checkout code"
+<<<<<<< HEAD
+        uses: actions/checkout@v4
+=======
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+>>>>>>> serverless-dns-main
         with:
           persist-credentials: false
 
       - name: "Run analysis"
+<<<<<<< HEAD
+        uses: ossf/[email protected]
+=======
         uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
+>>>>>>> serverless-dns-main
         with:
           results_file: results.sarif
           results_format: sarif

deno.Dockerfile

@@ -1,6 +1,6 @@
 # Based on github.com/denoland/deno_docker/blob/main/alpine.dockerfile
 
-ARG DENO_VERSION=1.29.2
+ARG DENO_VERSION=1.44.4
 ARG BIN_IMAGE=denoland/deno:bin-${DENO_VERSION}
 
 FROM ${BIN_IMAGE} AS bin

import_map.json

@@ -6,6 +6,7 @@
     "process": "https://deno.land/[email protected]/node/process.ts",
     "@serverless-dns/dns-parser": "https://github.com/serverless-dns/dns-parser/raw/v2.1.2/index.js",
     "@serverless-dns/lfu-cache": "https://github.com/serverless-dns/lfu-cache/raw/v3.4.1/lfu.js",
-    "@serverless-dns/trie/": "https://github.com/serverless-dns/trie/raw/v0.0.13/src/"
+    "@serverless-dns/trie/": "https://github.com/serverless-dns/trie/raw/v0.0.13/src/",
+    "@riaskov/mmap-io": "https://github.com/ARyaskov/mmap-io/raw/v1.4.3/src"
   }
 }

node.Dockerfile

@@ -1,27 +1,34 @@
-FROM node:20 as setup
+FROM node:22 as setup
 # git is required if any of the npm packages are git[hub] packages
 RUN apt-get update && apt-get install git -yq --no-install-suggests --no-install-recommends
-WORKDIR /node-dir
+WORKDIR /app
 COPY . .
 # get deps, build, bundle
 RUN npm i
+# webpack externalizes native modules (@riaskov/mmap-io)
 RUN npm run build:fly
 # or RUN npx webpack --config webpack.fly.cjs
 # download blocklists and bake them in the img
 RUN export BLOCKLIST_DOWNLOAD_ONLY=true && node ./dist/fly.mjs
+# or RUN export BLOCKLIST_DOWNLOAD_ONLY=true && node ./src/server-node.js
 
 # stage 2
-FROM node:alpine AS runner
+# pin to node22 for native deps (@ariaskov/mmap-io)
+FROM node:22-alpine AS runner
 
 # env vals persist even at run-time: archive.is/QpXp2
 # and overrides fly.toml env values
 ENV NODE_ENV production
 ENV NODE_OPTIONS="--max-old-space-size=320 --heapsnapshot-signal=SIGUSR2"
 # get working dir in order
 WORKDIR /app
-COPY --from=setup /node-dir/dist ./
-COPY --from=setup /node-dir/blocklists__ ./blocklists__
-COPY --from=setup /node-dir/dbip__ ./dbip__
+# external deps not bundled by webpack
+RUN npm i @riaskov/[email protected]
+
+COPY --from=setup /app/dist ./
+COPY --from=setup /app/blocklists__ ./blocklists__
+COPY --from=setup /app/dbip__ ./dbip__
+
 # print files in work dir, must contain blocklists
 RUN ls -Fla
 # run with the default entrypoint (usually, bash or sh)

src/commons/envutil.js

@@ -45,6 +45,11 @@ export function hasDisk() {
   return onFly() || onLocal();
 }
 
+export function useMmap() {
+  // got disk on fly and local deploys
+  return onFly() || onLocal();
+}
+
 export function hasDynamicImports() {
   if (onDenoDeploy() || onCloudflare() || onFastly()) return false;
   return true;

src/core/cfg.js

@@ -7,8 +7,9 @@
  */
 /* eslint-disabled */
 // eslint, no import-assert: github.com/eslint/eslint/discussions/15305
-import u6cfg from "../u6-basicconfig.json" assert { type: 'json' };
-import u6filetag from "../u6-filetag.json" assert { type: 'json' };
+import u6cfg from "../u6-basicconfig.json" with { type: 'json' };
+import u6filetag from "../u6-filetag.json" with { type: 'json' };
+// nodejs.org/docs/latest-v22.x/api/esm.html#json-modules
 
 export function timestamp() {
     return u6cfg.timestamp;
@@ -17,15 +18,15 @@ export function timestamp() {
 export function tdNodeCount() {
     return u6cfg.nodecount;
 }
-  
+
 export function tdParts() {
   return u6cfg.tdparts;
 }
 
 export function tdCodec6() {
   return u6cfg.useCodec6;
 }
-  
+
 export function orig() {
   return u6cfg;
 }

src/core/node/blocklists.js

@@ -10,6 +10,7 @@ import * as path from "node:path";
 import * as bufutil from "../../commons/bufutil.js";
 import * as envutil from "../../commons/envutil.js";
 import * as cfg from "../../core/cfg.js";
+import mmap from "@riaskov/mmap-io";
 
 const blocklistsDir = "./blocklists__";
 const tdFile = "td.txt";
@@ -26,8 +27,9 @@ export async function setup(bw) {
   const tdparts = cfg.tdParts();
   const tdcodec6 = cfg.tdCodec6();
   const codec = tdcodec6 ? "u6" : "u8";
+  const useMmap = envutil.useMmap();
 
-  const ok = setupLocally(bw, timestamp, codec);
+  const ok = setupLocally(bw, timestamp, codec, useMmap);
   if (ok) {
     log.i("bl setup locally tstamp/nc", timestamp, nodecount);
     return true;
@@ -64,15 +66,30 @@ function save(bw, timestamp, codec) {
   return true;
 }
 
-function setupLocally(bw, timestamp, codec) {
+// fmmap mmaps file at fp for random reads, returns a Buffer backed by the file.
+function fmmap(fp) {
+  const fd = fs.openSync(fp, "r+");
+  const fsize = fs.fstatSync(fd).size;
+  const rxprot = mmap.PROT_READ; // protection
+  const mpriv = mmap.MAP_SHARED; // privacy
+  const madv = mmap.MADV_RANDOM; // madvise
+  const offset = 0;
+  log.i("mmap f:", fp, "size:", fsize, "\nNOTE: md5 checks will fail");
+  return mmap.map(fsize, rxprot, mpriv, fd, offset, madv);
+}
+
+function setupLocally(bw, timestamp, codec, useMmap) {
   const ok = hasBlocklistFiles(timestamp, codec);
   log.i(timestamp, codec, "has bl files?", ok);
   if (!ok) return false;
 
   const [td, rd] = getFilePaths(timestamp, codec);
-  log.i("on-disk codec/td/rd", codec, td, rd);
+  log.i("on-disk codec/td/rd", codec, td, rd, "mmap?", useMmap);
 
-  const tdbuf = fs.readFileSync(td);
+  let tdbuf = useMmap ? fmmap(td) : null;
+  if (bufutil.emptyBuf(tdbuf)) {
+    tdbuf = fs.readFileSync(td);
+  }
   const rdbuf = fs.readFileSync(rd);
 
   // TODO: file integrity checks

webpack.fly.cjs

@@ -2,7 +2,7 @@ const webpack = require("webpack");
 
 module.exports = {
   entry: "./src/server-node.js",
-  target: ["node", "es2022"],
+  target: ["node22", "es2022"],
   mode: "production",
   // enable devtool in development
   // devtool: 'eval-cheap-module-source-map',
@@ -17,6 +17,12 @@ module.exports = {
     }),
   ],
 
+  /* externalsType: 'module',
+  externals: {
+    '@riaskov/mmap-io': '@riaskov/mmap-io',
+  },*/
+  externals: /@riaskov/,
+
   optimization: {
     usedExports: true,
     minimize: false,
@@ -47,7 +53,8 @@ module.exports = {
     module: true,
   },
 
-  /* or, cjs: stackoverflow.com/a/68916455
+  // or, cjs: stackoverflow.com/a/68916455
+  /*
   output: {
     filename: "fly.cjs",
     clean: true, // empty dist before output

wrangler.toml

@@ -8,6 +8,7 @@ logpush = false
 compatibility_date = "2023-03-21"
 send_metrics = false
 minify = false
+upload_source_maps = true
 
 # uncomment to enable analytics on serverless-dns
 # this binding is not inherited by other worker-envs