diff --git a/docker/from_source.Dockerfile b/docker/from_source.Dockerfile index 21f3aca743c4..767645a93499 100644 --- a/docker/from_source.Dockerfile +++ b/docker/from_source.Dockerfile @@ -30,6 +30,12 @@ RUN yarn install --non-interactive --frozen-lockfile --ignore-scripts COPY . . RUN yarn install --non-interactive --frozen-lockfile && yarn build +# To have access to the specific branch and commit used to build this source, +# a git-data.json file is created by persisting git data at build time. Then, +# a version string like `v0.35.0-beta.0/HEAD/82219149 (git)` can be shown in +# the terminal and in the logs; which is very useful to track tests better. +RUN cd packages/cli && yarn write-git-data + # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) FROM node:16-alpine diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 64278a9ec2d0..04595a475b27 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -3,9 +3,9 @@ import yargs from "yargs"; import {cmds} from "./cmds"; import {globalOptions, rcConfigOption} from "./options"; import {registerCommandToYargs} from "./util"; -import {getVersion} from "./util/version"; +import {getVersionData} from "./util/version"; -const version = getVersion(); +const {version} = getVersionData(); const topBanner = `🌟 Lodestar: TypeScript Implementation of the Ethereum Consensus Beacon Chain. * Version: ${version} * by ChainSafe Systems, 2018-2022`; diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index cd5c9695c4c8..91e5b335f04e 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -14,7 +14,7 @@ import {initializeOptionsAndConfig, persistOptionsAndConfig} from "../init/handl import {IBeaconArgs} from "./options"; import {getBeaconPaths} from "./paths"; import {initBeaconState} from "./initBeaconState"; -import {getVersion, getVersionGitData} from "../../util/version"; +import {getVersionData} from "../../util/version"; import {deleteOldPeerstorePreV036} from "../../migrations"; /** @@ -26,16 +26,15 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise { // BeaconNode setup const libp2p = await createNodeJsLibp2p(peerId, options.network, {peerStoreDir: beaconPaths.peerStoreDir}); const logger = getCliLogger(args, beaconPaths, config); - logger.info("Lodestar", {version: getVersion(), network: args.network}); + logger.info("Lodestar", {network: args.network, ...getVersionData()}); if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset"); const db = new BeaconDb({config, controller: new LevelDbController(options.db, {logger})}); diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index e86decd6a2c8..118e335e3c56 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -8,7 +8,7 @@ import {getBeaconConfigFromArgs} from "../../config"; import {IGlobalArgs} from "../../options"; import {YargsError, getDefaultGraffiti, initBLS, mkdir, getCliLogger} from "../../util"; import {onGracefulShutdown} from "../../util"; -import {getVersion, getVersionGitData} from "../../util/version"; +import {getVersionData} from "../../util/version"; import {getBeaconPaths} from "../beacon/paths"; import {getValidatorPaths} from "./paths"; import {IValidatorCliArgs, validatorMetricsDefaultOptions} from "./options"; @@ -28,9 +28,8 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P const logger = getCliLogger(args, beaconPaths, config); - const version = getVersion(); - const gitData = getVersionGitData(); - logger.info("Lodestar", {version: version, network: args.network}); + const {version, commit} = getVersionData(); + logger.info("Lodestar", {network: args.network, version, commit}); const dbPath = validatorPaths.validatorsDbDir; mkdir(dbPath); @@ -102,14 +101,7 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P const register = args["metrics.enabled"] ? new RegistryMetricCreator() : null; const metrics = - register && - getMetrics((register as unknown) as MetricsRegister, { - semver: gitData.semver ?? "-", - branch: gitData.branch ?? "-", - commit: gitData.commit ?? "-", - version, - network: args.network, - }); + register && getMetrics((register as unknown) as MetricsRegister, {version, commit, network: args.network}); // Start metrics server if metrics are enabled. // Collect NodeJS metrics defined in the Lodestar repo diff --git a/packages/cli/src/util/gitData/gitDataPath.ts b/packages/cli/src/util/gitData/gitDataPath.ts index 3a1e64d63622..85c0788bd9fa 100644 --- a/packages/cli/src/util/gitData/gitDataPath.ts +++ b/packages/cli/src/util/gitData/gitDataPath.ts @@ -1,11 +1,23 @@ -/** - * Persist git data and distribute through NPM so CLI consumers can know exactly - * at what commit was this src build. This is used in the metrics and to log initially. - */ - import path from "node:path"; import fs from "node:fs"; +// Persist git data and distribute through NPM so CLI consumers can know exactly +// at what commit was this src build. This is used in the metrics and to log initially. +// +// - For NPM release (stable): Only the version is persisted. Once must then track the version's tag +// in Github to resolve that version to a specific commit. While this is okay, git-data.json gives +// a gurantee of the exact commit at build time. +// +// - For NPM release (nightly): canary commits include the commit, so this feature is not really +// necessary. However, it's more cumbersome to have conditional logic on stable / nightly. +// +// - For build from source: .git folder is available in the context of the built code, so it can extract +// branch and commit directly without the need for .git-data.json. +// +// - For build from source dockerized: This feature is required to know the branch and commit, since +// git data is not persisted past the build. However, .dockerignore prevents .git folder from being +// copied into the container's context, so .git-data.json can't be generated. + /** * WARNING!! If you change this path make sure to update: * - 'packages/cli/package.json' -> .files -> `".git-data.json"` @@ -14,14 +26,10 @@ export const gitDataPath = path.resolve(__dirname, "../../../.git-data.json"); /** Git data type used to construct version information string and persistence. */ export type GitData = { - /** v0.28.2 */ - semver?: string; /** "developer-feature" */ - branch?: string; + branch: string; /** "80c248bb392f512cc115d95059e22239a17bbd7d" */ - commit?: string; - /** +7 (commits since last tag) */ - numCommits?: string; + commit: string; }; /** Writes a persistent git data file. */ diff --git a/packages/cli/src/util/gitData/index.ts b/packages/cli/src/util/gitData/index.ts index 9ab7b3cade3d..36a70a42bc72 100644 --- a/packages/cli/src/util/gitData/index.ts +++ b/packages/cli/src/util/gitData/index.ts @@ -1,108 +1,65 @@ import {execSync} from "node:child_process"; -/** - * This file is created in the build step and is distributed through NPM - * MUST be in sync with `-/gitDataPath.ts` and `package.json` files. - */ +// This file is created in the build step and is distributed through NPM +// MUST be in sync with `-/gitDataPath.ts` and `package.json` files. import {readGitDataFile, GitData} from "./gitDataPath"; -/** Silent shell that won't pollute stdout, or stderr */ -function shell(cmd: string): string { - return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]}) - .toString() - .trim(); -} - -/** Tries to get branch from git CLI. */ -function getBranch(): string | undefined { - try { - return shell("git rev-parse --abbrev-ref HEAD"); - } catch (e) { - return undefined; - } -} - -/** Tries to get commit from git from git CLI. */ -function getCommit(): string | undefined { - try { - return shell("git rev-parse --verify HEAD"); - } catch (e) { - return undefined; - } -} - -/** Tries to get the latest tag from git CLI. */ -function getLatestTag(): string | undefined { - try { - return shell("git describe --abbrev=0"); - } catch (e) { - return undefined; - } -} - -/** Gets number of commits since latest tag/release. */ -function getCommitsSinceRelease(): number | undefined { - let numCommits = 0; - const latestTag: string | undefined = getLatestTag(); - try { - numCommits = +shell(`git rev-list ${latestTag}..HEAD --count`); - } catch (e) { - return undefined; - } - return numCommits; -} - /** Reads git data from a persisted file or local git data at build time. */ -export function readLodestarGitData(): GitData { +export function readAndGetGitData(): GitData { try { - const currentGitData = getGitData(); - const persistedGitData = getPersistedGitData(); - - // If the CLI is run from source, prioritze current git data - // over `.git-data.json` file, which might be stale here. - let gitData = {...persistedGitData, ...currentGitData}; - - // If the CLI is not run from the git repository, fall back to persistent - if (!gitData.semver || !gitData.branch || !gitData.commit) { - gitData = persistedGitData; + // Gets git data containing current branch and commit info from persistent file. + let persistedGitData: Partial; + try { + persistedGitData = readGitDataFile(); + } catch (e) { + persistedGitData = {}; } + const currentGitData = getGitData(); + return { - semver: gitData?.semver, - branch: gitData?.branch || "N/A", - commit: gitData?.commit || "N/A", - numCommits: gitData?.numCommits || "", + // If the CLI is run from source, prioritze current git data + // over `.git-data.json` file, which might be stale here. + branch: currentGitData.branch ?? persistedGitData.branch ?? "", + commit: currentGitData.commit ?? persistedGitData.commit ?? "", }; } catch (e) { - return {semver: "", branch: "", commit: "", numCommits: ""}; + return { + branch: "", + commit: "", + }; } } -/** Wrapper for updating git data. ONLY to be used with build scripts! */ -export function forceUpdateGitData(): Partial { - return getGitData(); -} - /** Gets git data containing current branch and commit info from CLI. */ -function getGitData(): Partial { - const numCommits: number | undefined = getCommitsSinceRelease(); - let strCommits = ""; - if (numCommits !== undefined && numCommits > 0) { - strCommits = `+${numCommits}`; - } +export function getGitData(): GitData { return { branch: getBranch(), commit: getCommit(), - semver: getLatestTag(), - numCommits: strCommits, }; } -/** Gets git data containing current branch and commit info from persistent file. */ -function getPersistedGitData(): Partial { +/** Tries to get branch from git CLI. */ +function getBranch(): string { try { - return readGitDataFile(); + return shellSilent("git rev-parse --abbrev-ref HEAD"); } catch (e) { - return {}; + return ""; } } + +/** Tries to get commit from git from git CLI. */ +function getCommit(): string { + try { + return shellSilent("git rev-parse --verify HEAD"); + } catch (e) { + return ""; + } +} + +/** Silent shell that won't pollute stdout, or stderr */ +function shellSilent(cmd: string): string { + return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]}) + .toString() + .trim(); +} diff --git a/packages/cli/src/util/gitData/writeGitData.ts b/packages/cli/src/util/gitData/writeGitData.ts index 784dd181e25a..709d1580d18e 100644 --- a/packages/cli/src/util/gitData/writeGitData.ts +++ b/packages/cli/src/util/gitData/writeGitData.ts @@ -1,12 +1,10 @@ #!/usr/bin/env node -/** - * Persist git data and distribute through NPM so CLI consumers can know exactly - * at what commit was this source build. This is also used in the metrics and to log initially. - */ +// For RATIONALE of this file, check packages/cli/src/util/gitData/gitDataPath.ts +// Persist exact commit in NPM distributions for easier tracking of the build +import {getGitData} from "./index"; import {writeGitDataFile} from "./gitDataPath"; -import {forceUpdateGitData} from "./index"; -/** Script to write the git data file (json) used by the build procedures to persist git data. */ -writeGitDataFile(forceUpdateGitData()); +// Script to write the git data file (json) used by the build procedures to persist git data. +writeGitDataFile(getGitData()); diff --git a/packages/cli/src/util/graffiti.ts b/packages/cli/src/util/graffiti.ts index 6c7981813735..6812931ff9ce 100644 --- a/packages/cli/src/util/graffiti.ts +++ b/packages/cli/src/util/graffiti.ts @@ -1,4 +1,4 @@ -import {getVersion} from "./version"; +import {getVersionData} from "./version"; const lodestarPackageName = "Lodestar"; @@ -8,7 +8,7 @@ const lodestarPackageName = "Lodestar"; */ export function getDefaultGraffiti(): string { try { - const version = getVersion(); + const {version} = getVersionData(); return `${lodestarPackageName}-${version}`; } catch (e) { // eslint-disable-next-line no-console diff --git a/packages/cli/src/util/version.ts b/packages/cli/src/util/version.ts index 48e853b82348..4bd1de604fe4 100644 --- a/packages/cli/src/util/version.ts +++ b/packages/cli/src/util/version.ts @@ -1,57 +1,52 @@ import fs from "node:fs"; import findUp from "find-up"; -import {readLodestarGitData} from "./gitData"; -import {GitData} from "./gitData/gitDataPath"; +import {readAndGetGitData} from "./gitData"; type VersionJson = { /** "0.28.2" */ version: string; }; -enum ReleaseTrack { - git = "git", - npm = "npm", - nightly = "nightly", - alpha = "alpha", - beta = "beta", - rc = "release candidate", - stable = "stable", - lts = "long term support", -} - -/** Defines default release track, i.e., the "stability" of tag releases */ -const defaultReleaseTrack = ReleaseTrack.alpha; +const BRANCH_IGNORE = /^(HEAD|master|main)$/; /** * Gathers all information on package version including Git data. - * @returns a version string, e.g., `v0.28.2/developer-feature/+7/80c248bb (nightly)` + * @returns a version string, e.g. + * - Stable release: `v0.36.0/80c248bb` + * - Nightly release: `v0.36.0-dev.80c248bb/80c248bb` + * - Test branch: `v0.36.0/developer-feature/80c248bb` */ -export function getVersion(): string { - const gitData: GitData = readLodestarGitData(); - let semver: string | undefined = gitData.semver; - const numCommits: string | undefined = gitData.numCommits; - const commitSlice: string | undefined = gitData.commit?.slice(0, 8); - - // ansible github branch deployment returns no semver - semver = semver ?? `v${getLocalVersion()}`; +export function getVersionData(): { + version: string; + commit: string; +} { + const parts: string[] = []; - // Tag release has numCommits as "0" - if (!commitSlice || numCommits === "0") { - return `${semver} (${defaultReleaseTrack})`; + /** Returns local version from `lerna.json` or `package.json` as `"0.28.2"` */ + const localVersion = readCliPackageJson() || readVersionFromLernaJson(); + if (localVersion) { + parts.push(`v${localVersion}`); } - // Otherwise get branch and commit information - return `${semver}/${gitData.branch}/${numCommits}/${commitSlice} (${ReleaseTrack.git})`; -} + const {branch, commit} = readAndGetGitData(); -/** Exposes raw version data wherever needed for reporting (metrics, grafana). */ -export function getVersionGitData(): GitData { - return readLodestarGitData(); -} + // Add branch only if not present and not an ignore value + if (branch && !BRANCH_IGNORE.test(branch)) parts.push(branch); + + // Add commit only if present. 7 characters to be consistent with Github + if (commit) { + const commitShort = commit.slice(0, 7); + // Don't add commit if it's already in the version string (nightly versions) + if (!localVersion || !localVersion.includes(commitShort)) { + parts.push(commitShort); + } + } -/** Returns local version from `lerna.json` or `package.json` as `"0.28.2"` */ -function getLocalVersion(): string | undefined { - return readVersionFromLernaJson() || readCliPackageJson(); + return { + // Guard against empty parts array + version: parts.length > 0 ? parts.join("/") : "unknown", + commit, + }; } /** Read version information from lerna.json */ diff --git a/packages/cli/test/unit/util/gitData.test.ts b/packages/cli/test/unit/util/gitData.test.ts index 901e202d6b59..9cfa08456b5e 100644 --- a/packages/cli/test/unit/util/gitData.test.ts +++ b/packages/cli/test/unit/util/gitData.test.ts @@ -1,15 +1,28 @@ import {expect} from "chai"; +import child_process from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import findUp from "find-up"; import {gitDataPath, readGitDataFile} from "../../../src/util/gitData/gitDataPath"; +import {getGitData} from "../../../src/util"; + +const WRITE_GIT_DATA_CMD = "npm run write-git-data"; describe("util / gitData", () => { + before(() => { + const pkgJsonPath = findUp.sync("package.json", {cwd: __dirname}); + if (!pkgJsonPath) { + throw Error("No package.json found"); + } + + const pkgJsonDir = path.resolve(path.dirname(pkgJsonPath)); + child_process.execSync(WRITE_GIT_DATA_CMD, {cwd: pkgJsonDir}); + }); + it("gitData file must exist", () => { const gitData = readGitDataFile(); - if (!gitData.branch) throw Error("No gitData.branch"); - if (!gitData.commit) throw Error("No gitData.commit"); + expect(gitData).to.deep.equal(getGitData(), "Wrong git-data.json contents"); }); it("gitData path must be included in the package.json", () => { diff --git a/packages/lodestar/src/metrics/options.ts b/packages/lodestar/src/metrics/options.ts index 9a3d2e3495f6..a6f28e0501c7 100644 --- a/packages/lodestar/src/metrics/options.ts +++ b/packages/lodestar/src/metrics/options.ts @@ -5,14 +5,10 @@ import {HttpMetricsServerOpts} from "./server"; export type LodestarMetadata = { - /** "0.16.0" */ - semver: string; - /** "developer/feature-1" */ - branch: string; + /** "v0.16.0/developer/feature-1/ac99f2b5" */ + version: string; /** "4f816b16dfde718e2d74f95f2c8292596138c248" */ commit: string; - /** "0.16.0 developer/feature-1 ac99f2b5" */ - version: string; /** "prater" */ network: string; }; diff --git a/packages/validator/src/metrics.ts b/packages/validator/src/metrics.ts index 90015fefd808..877f33c9f6e2 100644 --- a/packages/validator/src/metrics.ts +++ b/packages/validator/src/metrics.ts @@ -60,14 +60,10 @@ export interface MetricsRegister { export type Metrics = ReturnType; export type LodestarGitData = { - /** "0.16.0" */ - semver: string; - /** "developer/feature-1" */ - branch: string; - /** "4f816b16dfde718e2d74f95f2c8292596138c248" */ - commit: string; /** "0.16.0 developer/feature-1 ac99f2b5" */ version: string; + /** "4f816b16dfde718e2d74f95f2c8292596138c248" */ + commit: string; /** "prater" */ network: string; };