From d1397789b8f06fb4cecb3445a15e8cda550bce64 Mon Sep 17 00:00:00 2001 From: annguyen0 Date: Sun, 4 Aug 2024 00:12:38 +0700 Subject: [PATCH] Merge (#197) * gh-action: https://github.com/ossf/scorecard-action/issues/997 * dockerfile: node 22 * gh-action: wrangler 3.56 * wrangler: upload source maps * fly: mmap to read trie on disk * node: rmv webpack for backend * use mmap for node 22 * node: type assertions with type accessors * node: use @aryaskov/mmap-io * deno: v1.44.4 * node: fix container cwd * gh-action: denoland/deployctl version * deno: import_map mmap-io * fly: webpack bundle target node 22 * fly: webpack externalize native module mmap * node: rmv unused var from blocklists.js * node: omit dev deps in docker * fly: bundle node_modules instead of copying brings down image size down from 900+mb (400mb+ due to node_modules, mostly devDependencies) to 300mb+ * fly: do not omit-dev in setup (dep: webpack) * fly: fix entrypoint * gh-action: ghcr for node-alpine mk1 --------- Co-authored-by: ignoramous Co-authored-by: Murtaza Aliakbar --- .github/workflows/cf.yml | 2 ++ .github/workflows/deno-deploy.yml | 2 +- .github/workflows/ghcr.yml | 60 +++++++++++++++++++++++++++++++ .github/workflows/scorecard.yml | 8 +++++ deno.Dockerfile | 2 +- import_map.json | 3 +- node.Dockerfile | 19 ++++++---- src/commons/envutil.js | 5 +++ src/core/cfg.js | 9 ++--- src/core/node/blocklists.js | 25 ++++++++++--- webpack.fly.cjs | 11 ++++-- wrangler.toml | 1 + 12 files changed, 128 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/ghcr.yml diff --git a/.github/workflows/cf.yml b/.github/workflows/cf.yml index 1b3fba872f..023fbd66c4 100644 --- a/.github/workflows/cf.yml +++ b/.github/workflows/cf.yml @@ -52,6 +52,7 @@ on: env: GIT_REF: ${{ github.event.inputs.commit || github.ref }} + WRANGLER_VER: '3.56.0' # default is 'dev' which is really empty/no env WORKERS_ENV: '' @@ -118,6 +119,7 @@ jobs: accountId: ${{ secrets.CF_ACCOUNT_ID }} # input overrides env-defaults, regardless environment: ${{ env.WORKERS_ENV }} + wranglerVersion: ${{ env.WRANGLER_VER }} env: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.NEXTDNS_CONF }} GIT_COMMIT_ID: ${{ env.COMMIT_SHA }} diff --git a/.github/workflows/deno-deploy.yml b/.github/workflows/deno-deploy.yml index 1af004bcb2..6d47c94e87 100644 --- a/.github/workflows/deno-deploy.yml +++ b/.github/workflows/deno-deploy.yml @@ -96,7 +96,7 @@ jobs: - name: 🦕 Install Deno @1.29 uses: denoland/setup-deno@5fae568d37c3b73449009674875529a984555dd1 # main with: - deno-version: 1.29.3 + deno-version: 1.44.4 - name: 📦 Bundle up if: ${{ env.DEPLOY_MODE == 'action' }} diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml new file mode 100644 index 0000000000..c3b5b3d733 --- /dev/null +++ b/.github/workflows/ghcr.yml @@ -0,0 +1,60 @@ +name: 🔄 runc + +on: + push: + tags: + - "v*" + workflow_dispatch: + +env: + REGISTRY: "ghcr.io" + IMAGE_NAME: ${{ github.repository }} + GIT_REF: ${{ github.event.inputs.git-ref || github.ref }} + +# docs.github.com/en/actions/publishing-packages/publishing-docker-images +jobs: + nodejs: + name: 🚀 Node on Alpine + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + - name: 🚚 Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.GIT_REF }} + fetch-depth: 0 + + - name: 🔐 Login + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: 🏷️ Metadata + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: 🛠 Build + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: ./node.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: 📕 Attest + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7547bce543..90839c7797 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -37,12 +37,20 @@ jobs: egress-policy: audit - name: "Checkout code" +<<<<<<< HEAD + uses: actions/checkout@v4 +======= uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 +>>>>>>> serverless-dns-main with: persist-credentials: false - name: "Run analysis" +<<<<<<< HEAD + uses: ossf/scorecard-action@v2.3.1 +======= uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3 +>>>>>>> serverless-dns-main with: results_file: results.sarif results_format: sarif diff --git a/deno.Dockerfile b/deno.Dockerfile index a7a7d65e6c..d356478342 100644 --- a/deno.Dockerfile +++ b/deno.Dockerfile @@ -1,6 +1,6 @@ # Based on github.com/denoland/deno_docker/blob/main/alpine.dockerfile -ARG DENO_VERSION=1.29.2 +ARG DENO_VERSION=1.44.4 ARG BIN_IMAGE=denoland/deno:bin-${DENO_VERSION} FROM ${BIN_IMAGE} AS bin diff --git a/import_map.json b/import_map.json index ac37fef4f6..47bb695696 100644 --- a/import_map.json +++ b/import_map.json @@ -6,6 +6,7 @@ "process": "https://deno.land/std@0.177.0/node/process.ts", "@serverless-dns/dns-parser": "https://github.com/serverless-dns/dns-parser/raw/v2.1.2/index.js", "@serverless-dns/lfu-cache": "https://github.com/serverless-dns/lfu-cache/raw/v3.4.1/lfu.js", - "@serverless-dns/trie/": "https://github.com/serverless-dns/trie/raw/v0.0.13/src/" + "@serverless-dns/trie/": "https://github.com/serverless-dns/trie/raw/v0.0.13/src/", + "@riaskov/mmap-io": "https://github.com/ARyaskov/mmap-io/raw/v1.4.3/src" } } diff --git a/node.Dockerfile b/node.Dockerfile index 0a499f772d..0680f23584 100644 --- a/node.Dockerfile +++ b/node.Dockerfile @@ -1,17 +1,20 @@ -FROM node:20 as setup +FROM node:22 as setup # git is required if any of the npm packages are git[hub] packages RUN apt-get update && apt-get install git -yq --no-install-suggests --no-install-recommends -WORKDIR /node-dir +WORKDIR /app COPY . . # get deps, build, bundle RUN npm i +# webpack externalizes native modules (@riaskov/mmap-io) RUN npm run build:fly # or RUN npx webpack --config webpack.fly.cjs # download blocklists and bake them in the img RUN export BLOCKLIST_DOWNLOAD_ONLY=true && node ./dist/fly.mjs +# or RUN export BLOCKLIST_DOWNLOAD_ONLY=true && node ./src/server-node.js # stage 2 -FROM node:alpine AS runner +# pin to node22 for native deps (@ariaskov/mmap-io) +FROM node:22-alpine AS runner # env vals persist even at run-time: archive.is/QpXp2 # and overrides fly.toml env values @@ -19,9 +22,13 @@ ENV NODE_ENV production ENV NODE_OPTIONS="--max-old-space-size=320 --heapsnapshot-signal=SIGUSR2" # get working dir in order WORKDIR /app -COPY --from=setup /node-dir/dist ./ -COPY --from=setup /node-dir/blocklists__ ./blocklists__ -COPY --from=setup /node-dir/dbip__ ./dbip__ +# external deps not bundled by webpack +RUN npm i @riaskov/mmap-io@v1.4.3 + +COPY --from=setup /app/dist ./ +COPY --from=setup /app/blocklists__ ./blocklists__ +COPY --from=setup /app/dbip__ ./dbip__ + # print files in work dir, must contain blocklists RUN ls -Fla # run with the default entrypoint (usually, bash or sh) diff --git a/src/commons/envutil.js b/src/commons/envutil.js index b270992d3a..834fb3c31e 100644 --- a/src/commons/envutil.js +++ b/src/commons/envutil.js @@ -45,6 +45,11 @@ export function hasDisk() { return onFly() || onLocal(); } +export function useMmap() { + // got disk on fly and local deploys + return onFly() || onLocal(); +} + export function hasDynamicImports() { if (onDenoDeploy() || onCloudflare() || onFastly()) return false; return true; diff --git a/src/core/cfg.js b/src/core/cfg.js index cc24ae7e82..5d30d00221 100644 --- a/src/core/cfg.js +++ b/src/core/cfg.js @@ -7,8 +7,9 @@ */ /* eslint-disabled */ // eslint, no import-assert: github.com/eslint/eslint/discussions/15305 -import u6cfg from "../u6-basicconfig.json" assert { type: 'json' }; -import u6filetag from "../u6-filetag.json" assert { type: 'json' }; +import u6cfg from "../u6-basicconfig.json" with { type: 'json' }; +import u6filetag from "../u6-filetag.json" with { type: 'json' }; +// nodejs.org/docs/latest-v22.x/api/esm.html#json-modules export function timestamp() { return u6cfg.timestamp; @@ -17,7 +18,7 @@ export function timestamp() { export function tdNodeCount() { return u6cfg.nodecount; } - + export function tdParts() { return u6cfg.tdparts; } @@ -25,7 +26,7 @@ export function tdParts() { export function tdCodec6() { return u6cfg.useCodec6; } - + export function orig() { return u6cfg; } diff --git a/src/core/node/blocklists.js b/src/core/node/blocklists.js index 111914f53d..6cb7b3bfb3 100644 --- a/src/core/node/blocklists.js +++ b/src/core/node/blocklists.js @@ -10,6 +10,7 @@ import * as path from "node:path"; import * as bufutil from "../../commons/bufutil.js"; import * as envutil from "../../commons/envutil.js"; import * as cfg from "../../core/cfg.js"; +import mmap from "@riaskov/mmap-io"; const blocklistsDir = "./blocklists__"; const tdFile = "td.txt"; @@ -26,8 +27,9 @@ export async function setup(bw) { const tdparts = cfg.tdParts(); const tdcodec6 = cfg.tdCodec6(); const codec = tdcodec6 ? "u6" : "u8"; + const useMmap = envutil.useMmap(); - const ok = setupLocally(bw, timestamp, codec); + const ok = setupLocally(bw, timestamp, codec, useMmap); if (ok) { log.i("bl setup locally tstamp/nc", timestamp, nodecount); return true; @@ -64,15 +66,30 @@ function save(bw, timestamp, codec) { return true; } -function setupLocally(bw, timestamp, codec) { +// fmmap mmaps file at fp for random reads, returns a Buffer backed by the file. +function fmmap(fp) { + const fd = fs.openSync(fp, "r+"); + const fsize = fs.fstatSync(fd).size; + const rxprot = mmap.PROT_READ; // protection + const mpriv = mmap.MAP_SHARED; // privacy + const madv = mmap.MADV_RANDOM; // madvise + const offset = 0; + log.i("mmap f:", fp, "size:", fsize, "\nNOTE: md5 checks will fail"); + return mmap.map(fsize, rxprot, mpriv, fd, offset, madv); +} + +function setupLocally(bw, timestamp, codec, useMmap) { const ok = hasBlocklistFiles(timestamp, codec); log.i(timestamp, codec, "has bl files?", ok); if (!ok) return false; const [td, rd] = getFilePaths(timestamp, codec); - log.i("on-disk codec/td/rd", codec, td, rd); + log.i("on-disk codec/td/rd", codec, td, rd, "mmap?", useMmap); - const tdbuf = fs.readFileSync(td); + let tdbuf = useMmap ? fmmap(td) : null; + if (bufutil.emptyBuf(tdbuf)) { + tdbuf = fs.readFileSync(td); + } const rdbuf = fs.readFileSync(rd); // TODO: file integrity checks diff --git a/webpack.fly.cjs b/webpack.fly.cjs index bd339b6fc4..427b82ab3e 100644 --- a/webpack.fly.cjs +++ b/webpack.fly.cjs @@ -2,7 +2,7 @@ const webpack = require("webpack"); module.exports = { entry: "./src/server-node.js", - target: ["node", "es2022"], + target: ["node22", "es2022"], mode: "production", // enable devtool in development // devtool: 'eval-cheap-module-source-map', @@ -17,6 +17,12 @@ module.exports = { }), ], + /* externalsType: 'module', + externals: { + '@riaskov/mmap-io': '@riaskov/mmap-io', + },*/ + externals: /@riaskov/, + optimization: { usedExports: true, minimize: false, @@ -47,7 +53,8 @@ module.exports = { module: true, }, - /* or, cjs: stackoverflow.com/a/68916455 + // or, cjs: stackoverflow.com/a/68916455 + /* output: { filename: "fly.cjs", clean: true, // empty dist before output diff --git a/wrangler.toml b/wrangler.toml index 3f7af9375b..9d0e05f6d0 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -8,6 +8,7 @@ logpush = false compatibility_date = "2023-03-21" send_metrics = false minify = false +upload_source_maps = true # uncomment to enable analytics on serverless-dns # this binding is not inherited by other worker-envs