Merge branch 'master' into docs/authwit-how-to

.github/ensure-builder/action.yml

@@ -24,7 +24,7 @@ runs:
         echo "spot_strategy=BestEffort" >> $GITHUB_OUTPUT
         echo "runner_label=$USERNAME-$runner_type" >> $GITHUB_OUTPUT
         if [[ $TYPE == builder-x86 ]]; then
-          # 128-core x86 instance types with least evictions
+          # 128-core x86 instance types, aws chooses for us based on capacity
           echo "instance_type=i4i.32xlarge m6a.32xlarge m6i.32xlarge m6id.32xlarge m6idn.32xlarge m6in.32xlarge m7a.32xlarge r6a.32xlarge r6i.32xlarge r6id.32xlarge r6in.32xlarge r7iz.32xlarge" >> $GITHUB_OUTPUT
           echo "ami_id=ami-04d8422a9ba4de80f" >> $GITHUB_OUTPUT
           echo "ebs_cache_size=256" >> $GITHUB_OUTPUT

.github/ensure-tester/action.yml

@@ -61,7 +61,7 @@ runs:
         ec2_subnet_id: subnet-4cfabd25
         ec2_security_group_id: sg-0ccd4e5df0dcca0c9
         ec2_key_name: "build-instance"
-        ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]'
+        ec2_instance_tags: '[]'
 
     # Set up a context for this run
     - name: Copy Repo to Tester

.github/spot-runner-action/dist/index.js

@@ -327,7 +327,7 @@ class Ec2Instance {
                 LaunchTemplateConfigs: [fleetLaunchConfig],
                 ClientToken: clientToken,
                 SpotOptions: {
-                    AllocationStrategy: "capacity-optimized",
+                    AllocationStrategy: "price-capacity-optimized",
                 },
                 TargetCapacitySpecification: {
                     TotalTargetCapacity: 1,

.github/spot-runner-action/src/ec2.ts

@@ -246,7 +246,7 @@ export class Ec2Instance {
       LaunchTemplateConfigs: [fleetLaunchConfig],
       ClientToken: clientToken,
       SpotOptions: {
-        AllocationStrategy: "capacity-optimized",
+        AllocationStrategy: "price-capacity-optimized",
       },
       TargetCapacitySpecification: {
         TotalTargetCapacity: 1,

.github/workflows/ci-arm.yml

@@ -7,9 +7,7 @@ on:
   workflow_dispatch:
     inputs: {}
 concurrency:
-  # force parallelism in master, cancelling in branches (only relevant to workflow_dispatch)
-  group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}
-  cancel-in-progress: true
+  group: ci-master-arm
 env:
   DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}"
   RUN_ID: ${{ github.run_id }}

.github/workflows/ci.yml

@@ -153,8 +153,39 @@ jobs:
               --no-output \
               +${{ matrix.test }}
 
+  acir-bench:
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Setup and Test
+        uses: ./.github/ensure-tester-with-images
+        timeout-minutes: 40
+        with:
+          runner_type: 16core-tester-x86
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/barretenberg-acir-benches:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: cd noir && ../scripts/earthly-ci +export-bench-acir-bb
+          run: |
+            set -eux
+            cd ./noir/
+            export FORCE_COLOR=1
+            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
+            ../scripts/earthly-ci -P \
+              --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+              --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+              --no-output \
+              +bench-publish-acir-bb         
+
   bench-summary:
-    needs: bench-e2e
+    needs: 
+      - acir-bench
+      - bench-e2e
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
@@ -167,37 +198,33 @@ jobs:
           concurrency_key: build-x86
       - name: "Build and upload bench aggregate file"
         working-directory: ./yarn-project/scripts
-        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate
-      - name: "Download base benchmark"
+        run: |
+          earthly-ci -P \
+            --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+            --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+            +bench-aggregate
+      - name: "Download base benchmark and package into earthly"
         if: ${{ github.event_name == 'pull_request' }}
-        run: scripts/logs/download_base_benchmark_from_s3.sh
+        run: |
+          # Download the base benchmark locally (requires AWS creds and .git history)
+          mkdir -p $BENCH_FOLDER
+          ./scripts/logs/download_base_benchmark_from_s3.sh
+          # Package it into an earthly artifact to read from bench-comment
+          earthly-ci -P ./scripts/logs+pack-base-benchmark --LOCAL_BENCH_FOLDER=./tmp/bench
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          BENCH_FOLDER: ./yarn-project/scripts/bench
+          BENCH_FOLDER: "./scripts/logs/tmp/bench"
           PULL_REQUEST: "${{ github.event.pull_request.number }}"
       - name: "Generate summary comment if pull request"
         if: ${{ github.event_name == 'pull_request' }}
         working-directory: ./yarn-project/scripts
-        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment
-
-  noir-format:
-    needs: setup
-    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
-    steps:
-      - uses: actions/checkout@v4
-        with: { ref: "${{ env.GIT_COMMIT }}" }
-      - uses: ./.github/ci-setup-action
-        with:
-          concurrency_key: noir-format-x86
-      - name: "Format Noir"
-        working-directory: ./noir/
-        timeout-minutes: 40
-        run: earthly-ci --no-output ./+format
-      - name: "Format noir-projects"
-        working-directory: ./noir-projects/
-        timeout-minutes: 40
-        run: earthly-ci --no-output ./+format
+        run: |
+          earthly-ci -P \
+            --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+            --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+            --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} \
+            +bench-comment
 
   # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)
@@ -232,6 +259,24 @@ jobs:
         timeout-minutes: 40
         run: earthly-ci --no-output ./+test
 
+  noir-format:
+    needs: setup
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: noir-format-x86
+      - name: "Format Noir"
+        working-directory: ./noir/
+        timeout-minutes: 40
+        run: earthly-ci --no-output ./+format
+      - name: "Format noir-projects"
+        working-directory: ./noir-projects/
+        timeout-minutes: 40
+        run: earthly-ci --no-output ./+format
+
   noir-test:
     needs: setup
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
@@ -468,6 +513,7 @@ jobs:
       - setup
       - e2e
       - bb-native-tests
+      - acir-bench
       - bb-bench
       - yarn-project-formatting
       - yarn-project-test

.noir-sync-commit

@@ -1 +1 @@
-a5b7df12faf9d71ff24f8c5cde5e78da44558caf
+fb039f74df23aea39bc0593a5d538d82b4efadf0

.vscode/extensions.json

@@ -6,6 +6,8 @@
     "eamodio.gitlens",
     "ms-vsliveshare.vsliveshare",
     "juanblanco.solidity",
+    "noir-lang.vscode-noir",
+    "rust-lang.rust-analyzer",
     // Generate nicer Doxygen comments
     "cschlosser.doxdocgen",
     ///////////////////////////////////////

Earthfile

@@ -60,4 +60,4 @@ base-log-uploader:
         unzip awscliv2.zip && \
         ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update && \
         rm -rf aws awscliv2.zip
-    COPY +scripts/scripts /usr/src/scripts
+    COPY +scripts/scripts /usr/src/scripts

barretenberg/acir_tests/bench_acir_tests.sh

@@ -1,8 +1,14 @@
 #!/usr/bin/env bash
 
+cd "$(dirname "$0")"
+
 TEST_NAMES=("$@")
 THREADS=(1 4 16 32 64)
-BENCHMARKS=$(mktemp)
+BENCHMARKS=$LOG_FILE
+
+if [[ -z "${LOG_FILE}" ]]; then
+    BENCHMARKS=$(mktemp)
+fi
 
 if [ "${#TEST_NAMES[@]}" -eq 0 ]; then
     TEST_NAMES=(sha256 ecdsa_secp256k1 ecdsa_secp256r1 schnorr double_verify_proof)
@@ -64,4 +70,6 @@ function genthreadheaders(t, len,   res) {
 }
 '
 
-rm $BENCHMARKS
+if [[ -z "${LOG_FILE}" ]]; then
+    rm $BENCHMARKS
+fi

barretenberg/cpp/scripts/analyze_client_ivc_bench.py

@@ -13,7 +13,7 @@
     "Decider::construct_proof(t)",
     "ECCVMProver(CircuitBuilder&)(t)",
     "ECCVMProver::construct_proof(t)",
-    "GoblinTranslatorProver::construct_proof(t)",
+    "TranslatorProver::construct_proof(t)",
     "Goblin::merge(t)"
 ]
 with open(PREFIX/IVC_BENCH_JSON, "r") as read_file:

barretenberg/cpp/src/CMakeLists.txt

@@ -82,6 +82,7 @@ add_subdirectory(barretenberg/stdlib_circuit_builders)
 add_subdirectory(barretenberg/sumcheck)
 add_subdirectory(barretenberg/transcript)
 add_subdirectory(barretenberg/translator_vm)
+add_subdirectory(barretenberg/translator_vm_recursion)
 add_subdirectory(barretenberg/ultra_honk)
 add_subdirectory(barretenberg/vm)
 add_subdirectory(barretenberg/wasi)
@@ -146,6 +147,7 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:sumcheck_objects>
     $<TARGET_OBJECTS:transcript_objects>
     $<TARGET_OBJECTS:translator_vm_objects>
+    $<TARGET_OBJECTS:translator_vm_recursion_objects>
     $<TARGET_OBJECTS:ultra_honk_objects>
     $<TARGET_OBJECTS:vm_objects>)
 

barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/goblin.bench.cpp

@@ -109,7 +109,7 @@ BENCHMARK_DEFINE_F(GoblinBench, GoblinECCVMProve)(benchmark::State& state)
  * @brief Benchmark only the Translator component
  *
  */
-BENCHMARK_DEFINE_F(GoblinBench, GoblinTranslatorProve)(benchmark::State& state)
+BENCHMARK_DEFINE_F(GoblinBench, TranslatorProve)(benchmark::State& state)
 {
     Goblin goblin;
 
@@ -136,7 +136,7 @@ BENCHMARK_DEFINE_F(GoblinBench, GoblinTranslatorProve)(benchmark::State& state)
 BENCHMARK_REGISTER_F(GoblinBench, GoblinFull)->Unit(benchmark::kMillisecond)->ARGS;
 BENCHMARK_REGISTER_F(GoblinBench, GoblinAccumulate)->Unit(benchmark::kMillisecond)->ARGS;
 BENCHMARK_REGISTER_F(GoblinBench, GoblinECCVMProve)->Unit(benchmark::kMillisecond)->ARGS;
-BENCHMARK_REGISTER_F(GoblinBench, GoblinTranslatorProve)->Unit(benchmark::kMillisecond)->ARGS;
+BENCHMARK_REGISTER_F(GoblinBench, TranslatorProve)->Unit(benchmark::kMillisecond)->ARGS;
 
 } // namespace
 

barretenberg/cpp/src/barretenberg/benchmark/relations_bench/relations.bench.cpp

@@ -2,7 +2,7 @@
 #include "barretenberg/protogalaxy/protogalaxy_prover.hpp"
 #include "barretenberg/stdlib_circuit_builders/goblin_ultra_flavor.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_flavor.hpp"
-#include "barretenberg/translator_vm/goblin_translator_flavor.hpp"
+#include "barretenberg/translator_vm/translator_flavor.hpp"
 #include <benchmark/benchmark.h>
 
 namespace {
@@ -103,12 +103,12 @@ BENCHMARK(execute_relation_for_values<GoblinUltraFlavor, Poseidon2ExternalRelati
 BENCHMARK(execute_relation_for_values<GoblinUltraFlavor, Poseidon2InternalRelation<Fr>>);
 
 // Translator VM
-BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorDecompositionRelation<Fr>>);
-BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorOpcodeConstraintRelation<Fr>>);
-BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorAccumulatorTransferRelation<Fr>>);
-BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorDeltaRangeConstraintRelation<Fr>>);
-BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorNonNativeFieldRelation<Fr>>);
-BENCHMARK(execute_relation_for_values<GoblinTranslatorFlavor, GoblinTranslatorPermutationRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<TranslatorFlavor, TranslatorDecompositionRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<TranslatorFlavor, TranslatorOpcodeConstraintRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<TranslatorFlavor, TranslatorAccumulatorTransferRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<TranslatorFlavor, TranslatorDeltaRangeConstraintRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<TranslatorFlavor, TranslatorNonNativeFieldRelation<Fr>>);
+BENCHMARK(execute_relation_for_values<TranslatorFlavor, TranslatorPermutationRelation<Fr>>);
 
 // ECCVM
 BENCHMARK(execute_relation_for_values<ECCVMFlavor, ECCVMLookupRelation<Fq>>);

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp

@@ -119,7 +119,7 @@ TEST_F(ClientIVCTests, BasicFailure)
  * @brief Prove and verify accumulation of an arbitrary set of circuits
  *
  */
-TEST_F(ClientIVCTests, BasicLarge)
+TEST_F(ClientIVCTests, DISABLED_BasicLarge)
 {
     ClientIVC ivc;
 
@@ -142,7 +142,7 @@ TEST_F(ClientIVCTests, BasicLarge)
  * @brief Using a structured trace allows for the accumulation of circuits of varying size
  *
  */
-TEST_F(ClientIVCTests, BasicStructured)
+TEST_F(ClientIVCTests, DISABLED_BasicStructured)
 {
     ClientIVC ivc;
     ivc.structured_flag = true;

barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp

@@ -198,7 +198,7 @@ template <typename PCS> class ZeroMorphProver_ {
      *
      * and concatenation_term = \sum_{i=0}^{num_chunks_per_group}(x^{i * min_N + 1}concatenation_groups_batched_{i})
      *
-     * @note The concatenation term arises from an implementation detail in the Goblin Translator and is not part of the
+     * @note The concatenation term arises from an implementation detail in the Translator and is not part of the
      * conventional ZM protocol
      * @param input_polynomial
      * @param quotients
@@ -250,7 +250,7 @@ template <typename PCS> class ZeroMorphProver_ {
         // If necessary, add to Z_x the contribution related to concatenated polynomials:
         // \sum_{i=0}^{num_chunks_per_group}(x^{i * min_n + 1}concatenation_groups_batched_{i}).
         // We are effectively reconstructing concatenated polynomials from their chunks now that we know x
-        // Note: this is an implementation detail related to Goblin Translator and is not part of the standard protocol.
+        // Note: this is an implementation detail related to Translator and is not part of the standard protocol.
         if (!concatenation_groups_batched.empty()) {
             size_t MINICIRCUIT_N = N / concatenation_groups_batched.size();
             auto x_to_minicircuit_N =
@@ -507,7 +507,7 @@ template <typename PCS> class ZeroMorphVerifier_ {
      *  concatenation_term = \sum{i=0}^{o-1}\sum_{j=0}^{num_chunks_per_group}(rho^{m+l+i} * x^{j * min_N + 1}
      *                       * concatenation_groups_commitments_{i}_{j})
      *
-     * @note The concatenation term arises from an implementation detail in the Goblin Translator and is not part of the
+     * @note The concatenation term arises from an implementation detail in the Translator and is not part of the
      * conventional ZM protocol
      * @param first_g1 first element in the SRS
      * @param f_commitments Commitments to unshifted polynomials [f_i]
@@ -565,7 +565,7 @@ template <typename PCS> class ZeroMorphVerifier_ {
         }
 
         // If applicable, add contribution from concatenated polynomial commitments
-        // Note: this is an implementation detail related to Goblin Translator and is not part of the standard protocol.
+        // Note: this is an implementation detail related to Translator and is not part of the standard protocol.
         if (!concatenation_groups_commitments.empty()) {
             size_t CONCATENATION_GROUP_SIZE = concatenation_groups_commitments[0].size();
             size_t MINICIRCUIT_N = N / CONCATENATION_GROUP_SIZE;

barretenberg/cpp/src/barretenberg/common/benchmark.hpp

@@ -94,7 +94,7 @@ template <typename T, typename... Args> void write_benchmark(const std::string&
     }
     std::ostringstream oss;
     oss << "{\"timestamp\": \"" << getCurrentTimestamp() << "\", "
-        << "\"name\": \"" << name << "\", "
+        << "\"eventName\": \"" << name << "\", "
         << "\"type\": \"" << TypeTraits<T>::type << "\", "
         << "\"value\": " << value << ", "
         << "\"threads\": " << env_hardware_concurrency();

barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp

@@ -27,7 +27,8 @@ class ECCVMFlavor {
     using Curve = curve::Grumpkin;
     using G1 = typename Curve::Group;
     using PCS = IPA<Curve>;
-    using FF = typename G1::subgroup_field;
+    using FF = typename Curve::ScalarField;
+    using BF = typename Curve::BaseField;
     using Polynomial = bb::Polynomial<FF>;
     using GroupElement = typename G1::element;
     using Commitment = typename G1::affine_element;

barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.hpp

@@ -13,19 +13,20 @@ namespace bb {
 // We won't compile this class with Standard, but we will like want to compile it (at least for testing)
 // with a flavor that uses the curve Grumpkin, or a flavor that does/does not have zk, etc.
 class ECCVMProver {
+  public:
     using Flavor = ECCVMFlavor;
     using FF = typename Flavor::FF;
+    using BF = typename Flavor::BF;
     using PCS = typename Flavor::PCS;
     using CommitmentKey = typename Flavor::CommitmentKey;
     using ProvingKey = typename Flavor::ProvingKey;
     using Polynomial = typename Flavor::Polynomial;
     using CommitmentLabels = typename Flavor::CommitmentLabels;
     using Transcript = typename Flavor::Transcript;
-    using TranslationEvaluations = bb::TranslationEvaluations;
+    using TranslationEvaluations = bb::TranslationEvaluations_<FF, BF>;
     using ZeroMorph = ZeroMorphProver_<PCS>;
     using CircuitBuilder = typename Flavor::CircuitBuilder;
 
-  public:
     explicit ECCVMProver(CircuitBuilder& builder,
                          const std::shared_ptr<Transcript>& transcript = std::make_shared<Transcript>());