Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: simulator split #11144

Merged
merged 38 commits into from
Jan 11, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
cb9e4f1
optionally handle rpc errors with 200 + err body
Thunkar Jan 7, 2025
bbdebd7
fix
Thunkar Jan 7, 2025
0ddc6b1
somehow bootstrap fast didn't catch this
Thunkar Jan 7, 2025
8b06cc2
wip
Thunkar Jan 8, 2025
b92d801
more chunking
Thunkar Jan 8, 2025
569e7c4
fmt
Thunkar Jan 8, 2025
e017a6d
fmt
Thunkar Jan 8, 2025
00109f1
Merge branch 'master' of github.com:AztecProtocol/aztec-packages into…
Thunkar Jan 8, 2025
54916d2
fixes
Thunkar Jan 8, 2025
dd9f9b5
refactor
Thunkar Jan 8, 2025
3ec9700
wip
Thunkar Jan 9, 2025
1218abc
comments from review
Thunkar Jan 9, 2025
b16aa74
fmt
Thunkar Jan 9, 2025
0a1ad96
Merge branch 'master' of github.com:AztecProtocol/aztec-packages into…
Thunkar Jan 9, 2025
746fb60
fmt
Thunkar Jan 9, 2025
a71d8c6
fixed test
Thunkar Jan 9, 2025
569bfa8
unified simulation and proving
Thunkar Jan 9, 2025
0026103
test data and fmt
Thunkar Jan 9, 2025
5d290f7
more fixes
Thunkar Jan 9, 2025
9e64d16
fmt
Thunkar Jan 9, 2025
92ffde3
Merge branch 'master' of github.com:AztecProtocol/aztec-packages into…
Thunkar Jan 9, 2025
813e9d9
Merge branch 'master' of github.com:AztecProtocol/aztec-packages into…
Thunkar Jan 9, 2025
a7152d7
fix test
Thunkar Jan 9, 2025
02e071b
split
Thunkar Jan 9, 2025
a2bb19d
fixed test and config
Thunkar Jan 9, 2025
24fc409
Merge branch 'master' of github.com:AztecProtocol/aztec-packages into…
Thunkar Jan 9, 2025
78d85b0
merge fixes
Thunkar Jan 10, 2025
a518022
fmt
Thunkar Jan 10, 2025
6885a71
Merge branch 'master' of github.com:AztecProtocol/aztec-packages into…
Thunkar Jan 10, 2025
179cf6f
fmt
Thunkar Jan 10, 2025
4a9aac8
fixes
Thunkar Jan 10, 2025
d843731
fmt
Thunkar Jan 10, 2025
3bd3555
fix imports
Thunkar Jan 10, 2025
832788a
Merge branch 'master' into gj/simulator_split
Thunkar Jan 10, 2025
ee1d34d
dep cleanup
Thunkar Jan 10, 2025
4f32b8d
Merge branch 'master' into gj/simulator_split
Thunkar Jan 10, 2025
30553d5
Merge branch 'master' into gj/simulator_split
Thunkar Jan 10, 2025
b3287ee
Merge branch 'master' into gj/simulator_split
Thunkar Jan 11, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions boxes/boxes/vite/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@
"@aztec/key-store": "link:../../../yarn-project/key-store",
"@aztec/kv-store": "portal:../../../yarn-project/kv-store",
"@aztec/pxe": "link:../../../yarn-project/pxe",
"@noir-lang/acvm_js": "link:../../../noir/packages/acvm_js",
"@noir-lang/noirc_abi": "link:../../../noir/packages/noirc_abi",
"buffer": "^6.0.3",
"@aztec/simulator": "link:../../../yarn-project/simulator",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-toastify": "^10.0.6"
Expand Down
24 changes: 2 additions & 22 deletions boxes/boxes/vite/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,13 @@ import { ToastContainer } from "react-toastify";
import "react-toastify/dist/ReactToastify.css";
import "./App.css";
import { Home } from "./pages/home";
import { useEffect, useState } from "react";
import initACVM from "@noir-lang/acvm_js/web/acvm_js";
import initABI from "@noir-lang/noirc_abi/web/noirc_abi_wasm";
import acvmURL from "@noir-lang/acvm_js/web/acvm_js_bg.wasm?url";
import abiURL from "@noir-lang/noirc_abi/web/noirc_abi_wasm_bg.wasm?url";

const InitWasm = ({ children }: any) => {
const [init, setInit] = useState(false);
useEffect(() => {
(async () => {
await Promise.all([
initACVM(new URL(acvmURL, import.meta.url).toString()),
initABI(new URL(abiURL, import.meta.url).toString()),
]);
setInit(true);
})();
}, []);

return <div>{init && children}</div>;
};

function App() {
return (
<InitWasm>
<>
<Home />
<ToastContainer />
</InitWasm>
</>
);
}

Expand Down
41 changes: 18 additions & 23 deletions boxes/boxes/vite/src/config.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import {
AztecNode,
Fr,
createLogger,
deriveMasterIncomingViewingSecretKey,
Expand All @@ -12,10 +11,10 @@ import { PXEService } from "@aztec/pxe/service";
import { PXEServiceConfig, getPXEServiceConfig } from "@aztec/pxe/config";
import { KVPxeDatabase } from "@aztec/pxe/database";
import { KeyStore } from "@aztec/key-store";
import { PrivateKernelProver } from "@aztec/circuit-types";
import { L2TipsStore } from "@aztec/kv-store/stores";
import { createStore } from "@aztec/kv-store/indexeddb";
import { BBWASMLazyPrivateKernelProver } from "@aztec/bb-prover/wasm/lazy";
import { WASMSimulator } from "@aztec/simulator/client";

process.env = Object.keys(import.meta.env).reduce((acc, key) => {
acc[key.replace("VITE_", "")] = import.meta.env[key];
Expand All @@ -39,25 +38,11 @@ export class PrivateEnv {
config.dataDirectory = "pxe";
config.proverEnabled = true;
const aztecNode = await createAztecNodeClient(this.nodeURL);
const proofCreator = new BBWASMLazyPrivateKernelProver(16);
this.pxe = await this.createPXEService(aztecNode, config, proofCreator);
const encryptionPrivateKey = deriveMasterIncomingViewingSecretKey(
this.secretKey,
const simulationProvider = new WASMSimulator();
const proofCreator = new BBWASMLazyPrivateKernelProver(
simulationProvider,
16,
);
this.accountContract = new SchnorrAccountContract(encryptionPrivateKey);
this.account = new AccountManager(
this.pxe,
this.secretKey,
this.accountContract,
);
await this.account.deploy().wait();
}

async createPXEService(
aztecNode: AztecNode,
config: PXEServiceConfig,
proofCreator?: PrivateKernelProver,
) {
const l1Contracts = await aztecNode.getL1ContractAddresses();
const configWithContracts = {
...config,
Expand All @@ -75,16 +60,26 @@ export class PrivateEnv {
const db = await KVPxeDatabase.create(store);
const tips = new L2TipsStore(store, "pxe");

const pxe = new PXEService(
this.pxe = new PXEService(
keyStore,
aztecNode,
db,
tips,
proofCreator,
simulationProvider,
config,
);
await pxe.init();
return pxe;
await this.pxe.init();
const encryptionPrivateKey = deriveMasterIncomingViewingSecretKey(
this.secretKey,
);
this.accountContract = new SchnorrAccountContract(encryptionPrivateKey);
this.account = new AccountManager(
this.pxe,
this.secretKey,
this.accountContract,
);
await this.account.deploy().wait();
}

async getWallet() {
Expand Down
5 changes: 2 additions & 3 deletions boxes/boxes/vite/vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export default defineConfig({
allow: [
searchForWorkspaceRoot(process.cwd()),
"../../../yarn-project/noir-protocol-circuits-types/artifacts",
"../../../noir/packages/noirc_abi/web",
"../../../noir/packages/acvm_js/web",
],
},
},
Expand All @@ -51,7 +53,4 @@ export default defineConfig({
},
},
},
optimizeDeps: {
exclude: ["@noir-lang/acvm_js", "@noir-lang/noirc_abi", "@aztec/bb-prover"],
},
});
22 changes: 7 additions & 15 deletions boxes/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,12 @@ __metadata:
languageName: unknown
linkType: soft

"@aztec/simulator@link:../../../yarn-project/simulator::locator=vite%40workspace%3Aboxes%2Fvite":
version: 0.0.0-use.local
resolution: "@aztec/simulator@link:../../../yarn-project/simulator::locator=vite%40workspace%3Aboxes%2Fvite"
languageName: node
linkType: soft

"@aztec/types@link:../yarn-project/types::locator=aztec-app%40workspace%3A.":
version: 0.0.0-use.local
resolution: "@aztec/types@link:../yarn-project/types::locator=aztec-app%40workspace%3A."
Expand Down Expand Up @@ -1664,18 +1670,6 @@ __metadata:
languageName: node
linkType: hard

"@noir-lang/acvm_js@link:../../../noir/packages/acvm_js::locator=vite%40workspace%3Aboxes%2Fvite":
version: 0.0.0-use.local
resolution: "@noir-lang/acvm_js@link:../../../noir/packages/acvm_js::locator=vite%40workspace%3Aboxes%2Fvite"
languageName: node
linkType: soft

"@noir-lang/noirc_abi@link:../../../noir/packages/noirc_abi::locator=vite%40workspace%3Aboxes%2Fvite":
version: 0.0.0-use.local
resolution: "@noir-lang/noirc_abi@link:../../../noir/packages/noirc_abi::locator=vite%40workspace%3Aboxes%2Fvite"
languageName: node
linkType: soft

"@nolyfill/is-core-module@npm:1.0.39":
version: 1.0.39
resolution: "@nolyfill/is-core-module@npm:1.0.39"
Expand Down Expand Up @@ -12162,13 +12156,11 @@ __metadata:
"@aztec/key-store": "link:../../../yarn-project/key-store"
"@aztec/kv-store": "portal:../../../yarn-project/kv-store"
"@aztec/pxe": "link:../../../yarn-project/pxe"
"@aztec/simulator": "link:../../../yarn-project/simulator"
"@eslint/js": "npm:^9.13.0"
"@noir-lang/acvm_js": "link:../../../noir/packages/acvm_js"
"@noir-lang/noirc_abi": "link:../../../noir/packages/noirc_abi"
"@types/react": "npm:^18.3.12"
"@types/react-dom": "npm:^18.3.1"
"@vitejs/plugin-react-swc": "npm:^3.7.2"
buffer: "npm:^6.0.3"
eslint: "npm:^9.13.0"
eslint-plugin-react-hooks: "npm:^5.1.0"
eslint-plugin-react-refresh: "npm:^0.4.16"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,6 @@ To write a contract:

```rust
#include_code declaration /noir-projects/noir-contracts/contracts/easy_private_voting_contract/src/main.nr raw

// Imports

// Storage

// Functions
}
```

2. Define imports in your contract block
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/aztec-node/src/aztec-node/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ import {
createValidatorForAcceptingTxs,
getDefaultAllowedSetupFunctions,
} from '@aztec/sequencer-client';
import { PublicProcessorFactory } from '@aztec/simulator';
import { PublicProcessorFactory } from '@aztec/simulator/server';
import { Attributes, type TelemetryClient, type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client';
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
import { createValidatorClient } from '@aztec/validator-client';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { type ClientIvcProof } from '@aztec/circuits.js';
import { runInDirectory } from '@aztec/foundation/fs';
import { type Logger, createLogger } from '@aztec/foundation/log';
import { BundleArtifactProvider } from '@aztec/noir-protocol-circuits-types/client/bundle';
import { type SimulationProvider } from '@aztec/simulator/server';

import { encode } from '@msgpack/msgpack';
import { serializeWitness } from '@noir-lang/noirc_abi';
Expand All @@ -22,14 +23,21 @@ export class BBNativePrivateKernelProver extends BBPrivateKernelProver {
private bbBinaryPath: string,
private bbWorkingDirectory: string,
private skipCleanup: boolean,
protected override simulationProvider: SimulationProvider,
protected override log = createLogger('bb-prover:native'),
) {
super(new BundleArtifactProvider(), log);
super(new BundleArtifactProvider(), simulationProvider, log);
}

public static async new(config: BBConfig, log?: Logger) {
public static async new(config: BBConfig, simulationProvider: SimulationProvider, log?: Logger) {
await fs.mkdir(config.bbWorkingDirectory, { recursive: true });
return new BBNativePrivateKernelProver(config.bbBinaryPath, config.bbWorkingDirectory, !!config.bbSkipCleanup, log);
return new BBNativePrivateKernelProver(
config.bbBinaryPath,
config.bbWorkingDirectory,
!!config.bbSkipCleanup,
simulationProvider,
log,
);
}

private async _createClientIvcProof(
Expand Down
14 changes: 8 additions & 6 deletions yarn-project/bb-prover/src/prover/bb_private_kernel_prover.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,19 @@ import {
} from '@aztec/noir-protocol-circuits-types/client';
import { type ArtifactProvider, type ClientProtocolArtifact } from '@aztec/noir-protocol-circuits-types/types';
import { ClientCircuitVks } from '@aztec/noir-protocol-circuits-types/vks';
import { WASMSimulator } from '@aztec/simulator/client';
import { type SimulationProvider } from '@aztec/simulator/client';
import { type NoirCompiledCircuit } from '@aztec/types/noir';

import { type Abi, type WitnessMap } from '@noir-lang/types';

import { mapProtocolArtifactNameToCircuitName } from '../stats.js';

export abstract class BBPrivateKernelProver implements PrivateKernelProver {
protected simulator = new WASMSimulator();

constructor(protected artifactProvider: ArtifactProvider, protected log = createLogger('bb-prover')) {}
constructor(
protected artifactProvider: ArtifactProvider,
protected simulationProvider: SimulationProvider,
protected log = createLogger('bb-prover'),
) {}

public async generateInitOutput(
inputs: PrivateKernelInitCircuitPrivateInputs,
Expand Down Expand Up @@ -164,7 +166,7 @@ export abstract class BBPrivateKernelProver implements PrivateKernelProver {
const witnessMap = convertInputs(inputs, compiledCircuit.abi);

const timer = new Timer();
const outputWitness = await this.simulator.simulateCircuit(witnessMap, compiledCircuit);
const outputWitness = await this.simulationProvider.executeProtocolCircuit(witnessMap, compiledCircuit);
const output = convertOutputs(outputWitness, compiledCircuit.abi);

this.log.debug(`Simulated ${circuitType}`, {
Expand Down Expand Up @@ -194,7 +196,7 @@ export abstract class BBPrivateKernelProver implements PrivateKernelProver {

const witnessMap = convertInputs(inputs, compiledCircuit.abi);
const timer = new Timer();
const outputWitness = await this.simulator.simulateCircuit(witnessMap, compiledCircuit);
const outputWitness = await this.simulationProvider.executeProtocolCircuit(witnessMap, compiledCircuit);
const output = convertOutputs(outputWitness, compiledCircuit.abi);

this.log.debug(`Generated witness for ${circuitType}`, {
Expand Down
4 changes: 2 additions & 2 deletions yarn-project/bb-prover/src/prover/bb_prover.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ import {
convertSingleTxBlockRootRollupInputsToWitnessMap,
convertSingleTxBlockRootRollupOutputsFromWitnessMap,
} from '@aztec/noir-protocol-circuits-types/server';
import { NativeACVMSimulator } from '@aztec/simulator';
import { NativeACVMSimulator } from '@aztec/simulator/server';
import { Attributes, type TelemetryClient, trackSpan } from '@aztec/telemetry-client';

import { type WitnessMap } from '@noir-lang/types';
Expand Down Expand Up @@ -427,7 +427,7 @@ export class BBNativeRollupProver implements ServerCircuitProver {

const inputWitness = convertInput(input);
const timer = new Timer();
const outputWitness = await simulator.simulateCircuit(inputWitness, artifact);
const outputWitness = await simulator.executeProtocolCircuit(inputWitness, artifact);
const output = convertOutput(outputWitness);

const circuitName = mapProtocolArtifactNameToCircuitName(circuitType);
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/bb-prover/src/test/test_avm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import {
} from '@aztec/circuits.js';
import { computeVarArgsHash } from '@aztec/circuits.js/hash';
import { padArrayEnd } from '@aztec/foundation/collection';
import { type PublicFunctionCallResult } from '@aztec/simulator';
import { type PublicFunctionCallResult } from '@aztec/simulator/server';

// TODO: pub somewhere more usable - copied from abstract phase manager
export function getPublicInputs(result: PublicFunctionCallResult): PublicCircuitPublicInputs {
Expand Down
7 changes: 5 additions & 2 deletions yarn-project/bb-prover/src/test/test_circuit_prover.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ import {
convertSimulatedSingleTxBlockRootRollupOutputsFromWitnessMap,
} from '@aztec/noir-protocol-circuits-types/server';
import { ProtocolCircuitVks } from '@aztec/noir-protocol-circuits-types/vks';
import { type SimulationProvider, WASMSimulatorWithBlobs, emitCircuitSimulationStats } from '@aztec/simulator';
import { type SimulationProvider, WASMSimulatorWithBlobs, emitCircuitSimulationStats } from '@aztec/simulator/server';
import { type TelemetryClient, trackSpan } from '@aztec/telemetry-client';

import { type WitnessMap } from '@noir-lang/types';
Expand Down Expand Up @@ -324,7 +324,10 @@ export class TestCircuitProver implements ServerCircuitProver {
// the blob operations with an oracle. Appears to be no way to provide nativeACVM with a foreign call hander.
simulationProvider = this.wasmSimulator;
}
const witness = await simulationProvider.simulateCircuit(witnessMap, SimulatedServerCircuitArtifacts[artifactName]);
const witness = await simulationProvider.executeProtocolCircuit(
witnessMap,
SimulatedServerCircuitArtifacts[artifactName],
);

const result = convertOutput(witness);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { ClientIvcProof } from '@aztec/circuits.js';
import { createLogger } from '@aztec/foundation/log';
import { Timer } from '@aztec/foundation/timer';
import { type ArtifactProvider } from '@aztec/noir-protocol-circuits-types/types';
import { type SimulationProvider } from '@aztec/simulator/client';

import { serializeWitness } from '@noir-lang/noirc_abi';
import { type WitnessMap } from '@noir-lang/types';
Expand All @@ -13,10 +14,11 @@ import { BBPrivateKernelProver } from '../prover/bb_private_kernel_prover.js';
export abstract class BBWASMPrivateKernelProver extends BBPrivateKernelProver {
constructor(
protected override artifactProvider: ArtifactProvider,
protected override simulationProvider: SimulationProvider,
private threads: number = 1,
protected override log = createLogger('bb-prover:wasm'),
) {
super(artifactProvider, log);
super(artifactProvider, simulationProvider, log);
}

public override async createClientIvcProof(acirs: Buffer[], witnessStack: WitnessMap[]): Promise<ClientIvcProof> {
Expand Down
5 changes: 3 additions & 2 deletions yarn-project/bb-prover/src/wasm/bundle.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { createLogger } from '@aztec/foundation/log';
import { BundleArtifactProvider } from '@aztec/noir-protocol-circuits-types/client/bundle';
import { type SimulationProvider } from '@aztec/simulator/client';

import { BBWASMPrivateKernelProver } from './bb_wasm_private_kernel_prover.js';

export class BBWASMBundlePrivateKernelProver extends BBWASMPrivateKernelProver {
constructor(threads = 1, log = createLogger('bb-prover:wasm:bundle')) {
super(new BundleArtifactProvider(), threads, log);
constructor(simulationProvider: SimulationProvider, threads = 1, log = createLogger('bb-prover:wasm:bundle')) {
super(new BundleArtifactProvider(), simulationProvider, threads, log);
}
}
5 changes: 3 additions & 2 deletions yarn-project/bb-prover/src/wasm/lazy.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { createLogger } from '@aztec/foundation/log';
import { LazyArtifactProvider } from '@aztec/noir-protocol-circuits-types/client/lazy';
import { type SimulationProvider } from '@aztec/simulator/client';

import { BBWASMPrivateKernelProver } from './bb_wasm_private_kernel_prover.js';

export class BBWASMLazyPrivateKernelProver extends BBWASMPrivateKernelProver {
constructor(threads = 1, log = createLogger('bb-prover:wasm:lazy')) {
super(new LazyArtifactProvider(), threads, log);
constructor(simulationProvider: SimulationProvider, threads = 1, log = createLogger('bb-prover:wasm:lazy')) {
super(new LazyArtifactProvider(), simulationProvider, threads, log);
}
}
7 changes: 6 additions & 1 deletion yarn-project/end-to-end/src/e2e_block_building.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ import { TestContract } from '@aztec/noir-contracts.js/Test';
import { TokenContract } from '@aztec/noir-contracts.js/Token';
import { type SequencerClient, SequencerState } from '@aztec/sequencer-client';
import { type TestSequencerClient } from '@aztec/sequencer-client/test';
import { PublicProcessorFactory, type PublicTxResult, PublicTxSimulator, type WorldStateDB } from '@aztec/simulator';
import {
PublicProcessorFactory,
type PublicTxResult,
PublicTxSimulator,
type WorldStateDB,
} from '@aztec/simulator/server';
import { type TelemetryClient } from '@aztec/telemetry-client';
import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';

Expand Down
Loading
Loading