Skip to content

Commit

Permalink
Merge d542713 into 28e2c74
Browse files Browse the repository at this point in the history
  • Loading branch information
dapplion authored May 9, 2022
2 parents 28e2c74 + d542713 commit f453983
Show file tree
Hide file tree
Showing 13 changed files with 136 additions and 176 deletions.
6 changes: 6 additions & 0 deletions docker/from_source.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ RUN yarn install --non-interactive --frozen-lockfile --ignore-scripts
COPY . .
RUN yarn install --non-interactive --frozen-lockfile && yarn build

# To have access to the specific branch and commit used to build this source,
# a git-data.json file is created by persisting git data at build time. Then,
# a version string like `v0.35.0-beta.0/HEAD/82219149 (git)` can be shown in
# the terminal and in the logs; which is very useful to track tests better.
RUN cd packages/cli && yarn write-git-data

# Copy built src + node_modules to a new layer to prune unnecessary fs
# Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020)
FROM node:16-alpine
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ import yargs from "yargs";
import {cmds} from "./cmds";
import {globalOptions, rcConfigOption} from "./options";
import {registerCommandToYargs} from "./util";
import {getVersion} from "./util/version";
import {getVersionData} from "./util/version";

const version = getVersion();
const {version} = getVersionData();
const topBanner = `🌟 Lodestar: TypeScript Implementation of the Ethereum Consensus Beacon Chain.
* Version: ${version}
* by ChainSafe Systems, 2018-2022`;
Expand Down
11 changes: 5 additions & 6 deletions packages/cli/src/cmds/beacon/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {initializeOptionsAndConfig, persistOptionsAndConfig} from "../init/handl
import {IBeaconArgs} from "./options";
import {getBeaconPaths} from "./paths";
import {initBeaconState} from "./initBeaconState";
import {getVersion, getVersionGitData} from "../../util/version";
import {getVersionData} from "../../util/version";
import {deleteOldPeerstorePreV036} from "../../migrations";

/**
Expand All @@ -26,16 +26,15 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise<vo
const {beaconNodeOptions, config} = await initializeOptionsAndConfig(args);
await persistOptionsAndConfig(args);

const version = getVersion();
const gitData = getVersionGitData();
const {version, commit} = getVersionData();
const beaconPaths = getBeaconPaths(args);
// TODO: Rename db.name to db.path or db.location
beaconNodeOptions.set({db: {name: beaconPaths.dbDir}});
beaconNodeOptions.set({chain: {persistInvalidSszObjectsDir: beaconPaths.persistInvalidSszObjectsDir}});
// Add metrics metadata to show versioning + network info in Prometheus + Grafana
beaconNodeOptions.set({metrics: {metadata: {...gitData, version, network: args.network}}});
beaconNodeOptions.set({metrics: {metadata: {version, commit, network: args.network}}});
// Add detailed version string for API node/version endpoint
beaconNodeOptions.set({api: {version: version}});
beaconNodeOptions.set({api: {version}});

// ENR setup
const peerId = await readPeerId(beaconPaths.peerIdFile);
Expand All @@ -53,7 +52,7 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise<vo
abortController.abort();
}, logger.info.bind(logger));

logger.info("Lodestar", {version: version, network: args.network});
logger.info("Lodestar", {network: args.network, version, commit});
if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset");

// peerstore migration
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/cmds/dev/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {initializeOptionsAndConfig} from "../init/handler";
import {mkdir, initBLS, getCliLogger} from "../../util";
import {getBeaconPaths} from "../beacon/paths";
import {getValidatorPaths} from "../validator/paths";
import {getVersion} from "../../util/version";
import {getVersionData} from "../../util/version";

/**
* Run a beacon node with validator
Expand Down Expand Up @@ -68,7 +68,7 @@ export async function devHandler(args: IDevArgs & IGlobalArgs): Promise<void> {
// BeaconNode setup
const libp2p = await createNodeJsLibp2p(peerId, options.network, {peerStoreDir: beaconPaths.peerStoreDir});
const logger = getCliLogger(args, beaconPaths, config);
logger.info("Lodestar", {version: getVersion(), network: args.network});
logger.info("Lodestar", {network: args.network, ...getVersionData()});
if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset");

const db = new BeaconDb({config, controller: new LevelDbController(options.db, {logger})});
Expand Down
16 changes: 4 additions & 12 deletions packages/cli/src/cmds/validator/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {getBeaconConfigFromArgs} from "../../config";
import {IGlobalArgs} from "../../options";
import {YargsError, getDefaultGraffiti, initBLS, mkdir, getCliLogger} from "../../util";
import {onGracefulShutdown} from "../../util";
import {getVersion, getVersionGitData} from "../../util/version";
import {getVersionData} from "../../util/version";
import {getBeaconPaths} from "../beacon/paths";
import {getValidatorPaths} from "./paths";
import {IValidatorCliArgs, validatorMetricsDefaultOptions} from "./options";
Expand All @@ -28,9 +28,8 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P

const logger = getCliLogger(args, beaconPaths, config);

const version = getVersion();
const gitData = getVersionGitData();
logger.info("Lodestar", {version: version, network: args.network});
const {version, commit} = getVersionData();
logger.info("Lodestar", {network: args.network, version, commit});

const dbPath = validatorPaths.validatorsDbDir;
mkdir(dbPath);
Expand Down Expand Up @@ -102,14 +101,7 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P

const register = args["metrics.enabled"] ? new RegistryMetricCreator() : null;
const metrics =
register &&
getMetrics((register as unknown) as MetricsRegister, {
semver: gitData.semver ?? "-",
branch: gitData.branch ?? "-",
commit: gitData.commit ?? "-",
version,
network: args.network,
});
register && getMetrics((register as unknown) as MetricsRegister, {version, commit, network: args.network});

// Start metrics server if metrics are enabled.
// Collect NodeJS metrics defined in the Lodestar repo
Expand Down
30 changes: 19 additions & 11 deletions packages/cli/src/util/gitData/gitDataPath.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,23 @@
/**
* Persist git data and distribute through NPM so CLI consumers can know exactly
* at what commit was this src build. This is used in the metrics and to log initially.
*/

import path from "node:path";
import fs from "node:fs";

// Persist git data and distribute through NPM so CLI consumers can know exactly
// at what commit was this src build. This is used in the metrics and to log initially.
//
// - For NPM release (stable): Only the version is persisted. Once must then track the version's tag
// in Github to resolve that version to a specific commit. While this is okay, git-data.json gives
// a gurantee of the exact commit at build time.
//
// - For NPM release (nightly): canary commits include the commit, so this feature is not really
// necessary. However, it's more cumbersome to have conditional logic on stable / nightly.
//
// - For build from source: .git folder is available in the context of the built code, so it can extract
// branch and commit directly without the need for .git-data.json.
//
// - For build from source dockerized: This feature is required to know the branch and commit, since
// git data is not persisted past the build. However, .dockerignore prevents .git folder from being
// copied into the container's context, so .git-data.json can't be generated.

/**
* WARNING!! If you change this path make sure to update:
* - 'packages/cli/package.json' -> .files -> `".git-data.json"`
Expand All @@ -14,14 +26,10 @@ export const gitDataPath = path.resolve(__dirname, "../../../.git-data.json");

/** Git data type used to construct version information string and persistence. */
export type GitData = {
/** v0.28.2 */
semver?: string;
/** "developer-feature" */
branch?: string;
branch: string;
/** "80c248bb392f512cc115d95059e22239a17bbd7d" */
commit?: string;
/** +7 (commits since last tag) */
numCommits?: string;
commit: string;
};

/** Writes a persistent git data file. */
Expand Down
123 changes: 40 additions & 83 deletions packages/cli/src/util/gitData/index.ts
Original file line number Diff line number Diff line change
@@ -1,108 +1,65 @@
import {execSync} from "node:child_process";

/**
* This file is created in the build step and is distributed through NPM
* MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
*/
// This file is created in the build step and is distributed through NPM
// MUST be in sync with `-/gitDataPath.ts` and `package.json` files.
import {readGitDataFile, GitData} from "./gitDataPath";

/** Silent shell that won't pollute stdout, or stderr */
function shell(cmd: string): string {
return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
.toString()
.trim();
}

/** Tries to get branch from git CLI. */
function getBranch(): string | undefined {
try {
return shell("git rev-parse --abbrev-ref HEAD");
} catch (e) {
return undefined;
}
}

/** Tries to get commit from git from git CLI. */
function getCommit(): string | undefined {
try {
return shell("git rev-parse --verify HEAD");
} catch (e) {
return undefined;
}
}

/** Tries to get the latest tag from git CLI. */
function getLatestTag(): string | undefined {
try {
return shell("git describe --abbrev=0");
} catch (e) {
return undefined;
}
}

/** Gets number of commits since latest tag/release. */
function getCommitsSinceRelease(): number | undefined {
let numCommits = 0;
const latestTag: string | undefined = getLatestTag();
try {
numCommits = +shell(`git rev-list ${latestTag}..HEAD --count`);
} catch (e) {
return undefined;
}
return numCommits;
}

/** Reads git data from a persisted file or local git data at build time. */
export function readLodestarGitData(): GitData {
export function readAndGetGitData(): GitData {
try {
const currentGitData = getGitData();
const persistedGitData = getPersistedGitData();

// If the CLI is run from source, prioritze current git data
// over `.git-data.json` file, which might be stale here.
let gitData = {...persistedGitData, ...currentGitData};

// If the CLI is not run from the git repository, fall back to persistent
if (!gitData.semver || !gitData.branch || !gitData.commit) {
gitData = persistedGitData;
// Gets git data containing current branch and commit info from persistent file.
let persistedGitData: Partial<GitData>;
try {
persistedGitData = readGitDataFile();
} catch (e) {
persistedGitData = {};
}

const currentGitData = getGitData();

return {
semver: gitData?.semver,
branch: gitData?.branch || "N/A",
commit: gitData?.commit || "N/A",
numCommits: gitData?.numCommits || "",
// If the CLI is run from source, prioritze current git data
// over `.git-data.json` file, which might be stale here.
branch: currentGitData.branch ?? persistedGitData.branch ?? "",
commit: currentGitData.commit ?? persistedGitData.commit ?? "",
};
} catch (e) {
return {semver: "", branch: "", commit: "", numCommits: ""};
return {
branch: "",
commit: "",
};
}
}

/** Wrapper for updating git data. ONLY to be used with build scripts! */
export function forceUpdateGitData(): Partial<GitData> {
return getGitData();
}

/** Gets git data containing current branch and commit info from CLI. */
function getGitData(): Partial<GitData> {
const numCommits: number | undefined = getCommitsSinceRelease();
let strCommits = "";
if (numCommits !== undefined && numCommits > 0) {
strCommits = `+${numCommits}`;
}
export function getGitData(): GitData {
return {
branch: getBranch(),
commit: getCommit(),
semver: getLatestTag(),
numCommits: strCommits,
};
}

/** Gets git data containing current branch and commit info from persistent file. */
function getPersistedGitData(): Partial<GitData> {
/** Tries to get branch from git CLI. */
function getBranch(): string {
try {
return readGitDataFile();
return shellSilent("git rev-parse --abbrev-ref HEAD");
} catch (e) {
return {};
return "";
}
}

/** Tries to get commit from git from git CLI. */
function getCommit(): string {
try {
return shellSilent("git rev-parse --verify HEAD");
} catch (e) {
return "";
}
}

/** Silent shell that won't pollute stdout, or stderr */
function shellSilent(cmd: string): string {
return execSync(cmd, {stdio: ["ignore", "pipe", "ignore"]})
.toString()
.trim();
}
12 changes: 5 additions & 7 deletions packages/cli/src/util/gitData/writeGitData.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
#!/usr/bin/env node

/**
* Persist git data and distribute through NPM so CLI consumers can know exactly
* at what commit was this source build. This is also used in the metrics and to log initially.
*/
// For RATIONALE of this file, check packages/cli/src/util/gitData/gitDataPath.ts
// Persist exact commit in NPM distributions for easier tracking of the build

import {getGitData} from "./index";
import {writeGitDataFile} from "./gitDataPath";
import {forceUpdateGitData} from "./index";

/** Script to write the git data file (json) used by the build procedures to persist git data. */
writeGitDataFile(forceUpdateGitData());
// Script to write the git data file (json) used by the build procedures to persist git data.
writeGitDataFile(getGitData());
4 changes: 2 additions & 2 deletions packages/cli/src/util/graffiti.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {getVersion} from "./version";
import {getVersionData} from "./version";

const lodestarPackageName = "Lodestar";

Expand All @@ -8,7 +8,7 @@ const lodestarPackageName = "Lodestar";
*/
export function getDefaultGraffiti(): string {
try {
const version = getVersion();
const {version} = getVersionData();
return `${lodestarPackageName}-${version}`;
} catch (e) {
// eslint-disable-next-line no-console
Expand Down
Loading

0 comments on commit f453983

Please sign in to comment.