feat(avm): plumb externalcall hints (#6890)

And rewrite (de)serialization of hints.

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -514,22 +514,6 @@ void vk_as_fields(const std::string& vk_path, const std::string& output_path)
     }
 }
 
-avm_trace::ExecutionHints deserialize_execution_hints(const std::vector<uint8_t>& hints)
-{
-    avm_trace::ExecutionHints execution_hints;
-    if (hints.size() == 0) {
-        vinfo("no hints provided");
-    } else {
-        // Hints arrive serialised as a vector of <side effect counter, hint> pairs
-        using FF = avm_trace::FF;
-        std::vector<std::pair<FF, FF>> deser_hints = many_from_buffer<std::pair<FF, FF>>(hints);
-        for (auto& hint : deser_hints) {
-            execution_hints.side_effect_hints[static_cast<uint32_t>(hint.first)] = hint.second;
-        }
-    }
-    return execution_hints;
-}
-
 /**
  * @brief Writes an avm proof and corresponding (incomplete) verification key to files.
  *
@@ -553,13 +537,7 @@ void avm_prove(const std::filesystem::path& bytecode_path,
         bytecode_path.extension() == ".gz" ? gunzip(bytecode_path) : read_file(bytecode_path);
     std::vector<fr> const calldata = many_from_buffer<fr>(read_file(calldata_path));
     std::vector<fr> const public_inputs_vec = many_from_buffer<fr>(read_file(public_inputs_path));
-
-    avm_trace::ExecutionHints avm_hints;
-    try {
-        avm_hints = deserialize_execution_hints(read_file(hints_path));
-    } catch (std::runtime_error const& err) {
-        vinfo("No hints were provided for avm proving.... Might be fine!");
-    }
+    auto const avm_hints = bb::avm_trace::ExecutionHints::from(read_file(hints_path));
 
     // Hardcoded circuit size for now, with enough to support 16-bit range checks
     init_bn254_crs(1 << 17);

barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_common.hpp

@@ -6,6 +6,7 @@
 
 #include <array>
 #include <cstdint>
+#include <map>
 #include <unordered_map>
 
 namespace bb::avm_trace {
@@ -33,14 +34,32 @@ static const size_t AVM_TRACE_SIZE = 1 << 18;
 enum class IntermRegister : uint32_t { IA = 0, IB = 1, IC = 2, ID = 3 };
 enum class IndirectRegister : uint32_t { IND_A = 0, IND_B = 1, IND_C = 2, IND_D = 3 };
 
-// Keep following enum in sync with MAX_NEM_TAG below
+// Keep following enum in sync with MAX_MEM_TAG below
 enum class AvmMemoryTag : uint32_t { U0 = 0, U8 = 1, U16 = 2, U32 = 3, U64 = 4, U128 = 5, FF = 6 };
 static const uint32_t MAX_MEM_TAG = 6;
 
 static const size_t NUM_MEM_SPACES = 256;
 static const uint8_t INTERNAL_CALL_SPACE_ID = 255;
 static const uint32_t MAX_SIZE_INTERNAL_STACK = 1 << 16;
 
+struct ExternalCallHint {
+    FF success;
+    std::vector<FF> return_data;
+    FF l2_gas_used;
+    FF da_gas_used;
+};
+
+// Add support for deserialization of ExternalCallHint. This is implicitly used by serialize::read
+// when trying to read std::vector<ExternalCallHint>.
+inline void read(uint8_t const*& it, ExternalCallHint& hint)
+{
+    using serialize::read;
+    read(it, hint.success);
+    read(it, hint.return_data);
+    read(it, hint.l2_gas_used);
+    read(it, hint.da_gas_used);
+}
+
 struct ContractInstanceHint {
     FF instance_found_in_address;
     FF salt;
@@ -49,11 +68,86 @@ struct ContractInstanceHint {
     FF initialisation_hash;
     FF public_key_hash;
 };
+
+// Add support for deserialization of ContractInstanceHint.
+inline void read(uint8_t const*& it, ContractInstanceHint& hint)
+{
+    using serialize::read;
+    read(it, hint.instance_found_in_address);
+    read(it, hint.salt);
+    read(it, hint.deployer_addr);
+    read(it, hint.contract_class_id);
+    read(it, hint.initialisation_hash);
+    read(it, hint.public_key_hash);
+}
+
 struct ExecutionHints {
-    std::unordered_map<uint32_t, FF> side_effect_hints;
+    ExecutionHints() = default;
+    ExecutionHints(std::vector<std::pair<FF, FF>> storage_value_hints,
+                   std::vector<std::pair<FF, FF>> note_hash_exists_hints,
+                   std::vector<std::pair<FF, FF>> nullifier_exists_hints,
+                   std::vector<std::pair<FF, FF>> l1_to_l2_message_exists_hints,
+                   std::vector<ExternalCallHint> externalcall_hints,
+                   std::map<FF, ContractInstanceHint> contract_instance_hints)
+        : storage_value_hints(std::move(storage_value_hints))
+        , note_hash_exists_hints(std::move(note_hash_exists_hints))
+        , nullifier_exists_hints(std::move(nullifier_exists_hints))
+        , l1_to_l2_message_exists_hints(std::move(l1_to_l2_message_exists_hints))
+        , externalcall_hints(std::move(externalcall_hints))
+        , contract_instance_hints(std::move(contract_instance_hints))
+    {}
 
-    std::vector<std::vector<FF>> returndata_hints;
+    std::vector<std::pair<FF, FF>> storage_value_hints;
+    std::vector<std::pair<FF, FF>> note_hash_exists_hints;
+    std::vector<std::pair<FF, FF>> nullifier_exists_hints;
+    std::vector<std::pair<FF, FF>> l1_to_l2_message_exists_hints;
+    std::vector<ExternalCallHint> externalcall_hints;
+    // TODO(dbanks): not read yet.
     std::map<FF, ContractInstanceHint> contract_instance_hints;
+
+    static void push_vec_into_map(std::unordered_map<uint32_t, FF>& into_map,
+                                  const std::vector<std::pair<FF, FF>>& from_pair_vec)
+    {
+        for (const auto& pair : from_pair_vec) {
+            into_map[static_cast<uint32_t>(pair.first)] = pair.second;
+        }
+    }
+
+    // TODO: Cache.
+    // Side effect counter -> value
+    std::unordered_map<uint32_t, FF> get_side_effect_hints() const
+    {
+        std::unordered_map<uint32_t, FF> hints_map;
+        push_vec_into_map(hints_map, storage_value_hints);
+        push_vec_into_map(hints_map, note_hash_exists_hints);
+        push_vec_into_map(hints_map, nullifier_exists_hints);
+        push_vec_into_map(hints_map, l1_to_l2_message_exists_hints);
+        return hints_map;
+    }
+
+    static ExecutionHints from(const std::vector<uint8_t>& data)
+    {
+        std::vector<std::pair<FF, FF>> storage_value_hints;
+        std::vector<std::pair<FF, FF>> note_hash_exists_hints;
+        std::vector<std::pair<FF, FF>> nullifier_exists_hints;
+        std::vector<std::pair<FF, FF>> l1_to_l2_message_exists_hints;
+        // TODO(dbanks): not read yet.
+        std::map<FF, ContractInstanceHint> contract_instance_hints;
+
+        using serialize::read;
+        const auto* it = data.data();
+        read(it, storage_value_hints);
+        read(it, note_hash_exists_hints);
+        read(it, nullifier_exists_hints);
+        read(it, l1_to_l2_message_exists_hints);
+
+        std::vector<ExternalCallHint> externalcall_hints;
+        read(it, externalcall_hints);
+
+        return { std::move(storage_value_hints),    std::move(note_hash_exists_hints),
+                 std::move(nullifier_exists_hints), std::move(l1_to_l2_message_exists_hints),
+                 std::move(externalcall_hints),     std::move(contract_instance_hints) };
+    }
 };
 
 } // namespace bb::avm_trace

barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_trace.cpp

@@ -1387,7 +1387,7 @@ Row AvmTraceBuilder::create_kernel_output_opcode_with_set_metadata_output_from_h
     AvmMemTraceBuilder::MemRead read_a = mem_trace_builder.read_and_load_from_memory(
         call_ptr, clk, IntermRegister::IA, data_offset, AvmMemoryTag::FF, AvmMemoryTag::U8);
 
-    FF exists = execution_hints.side_effect_hints.at(side_effect_counter);
+    FF exists = execution_hints.get_side_effect_hints().at(side_effect_counter);
     // TODO: throw error if incorrect
 
     mem_trace_builder.write_into_memory(
@@ -1417,7 +1417,7 @@ Row AvmTraceBuilder::create_kernel_output_opcode_with_set_value_from_hint(uint32
                                                                           uint32_t data_offset,
                                                                           uint32_t metadata_offset)
 {
-    FF value = execution_hints.side_effect_hints.at(side_effect_counter);
+    FF value = execution_hints.get_side_effect_hints().at(side_effect_counter);
     // TODO: throw error if incorrect
 
     mem_trace_builder.write_into_memory(
@@ -2513,7 +2513,7 @@ void AvmTraceBuilder::op_call([[maybe_unused]] uint8_t indirect,
                           AvmMemoryTag::U0,
                           AvmMemoryTag::FF,
                           internal_return_ptr,
-                          execution_hints.returndata_hints.at(return_data_counter));
+                          execution_hints.externalcall_hints.at(return_data_counter).return_data);
     return_data_counter++;
     clk++;
     write_slice_to_memory(

barretenberg/cpp/src/barretenberg/vm/tests/avm_execution.test.cpp

@@ -1760,8 +1760,8 @@ TEST_F(AvmExecutionTests, kernelOutputStorageOpcodes)
     std::vector<FF> returndata = {};
 
     // Generate Hint for Sload operation
-    ExecutionHints execution_hints = {};
-    execution_hints.side_effect_hints[0] = FF(42); // side effect counter 0 = value 42
+    // side effect counter 0 = value 42
+    ExecutionHints execution_hints({ { 0, 42 } }, {}, {}, {}, {}, {});
 
     auto trace = Execution::gen_trace(instructions, returndata, calldata, public_inputs_vec, execution_hints);
 
@@ -1842,10 +1842,7 @@ TEST_F(AvmExecutionTests, kernelOutputHashExistsOpcodes)
     std::vector<FF> returndata = {};
 
     // Generate Hint for Sload operation
-    ExecutionHints execution_hints = {};
-    execution_hints.side_effect_hints[0] = 1; // Side effect counter 0 = true
-    execution_hints.side_effect_hints[1] = 1; // Side effect counter 1 = true
-    execution_hints.side_effect_hints[2] = 1; // Side effect counter 2 = true
+    ExecutionHints execution_hints({ { 0, 1 }, { 1, 1 }, { 2, 1 } }, {}, {}, {}, {}, {});
 
     auto trace = Execution::gen_trace(instructions, returndata, calldata, public_inputs_vec, execution_hints);
 
@@ -1953,8 +1950,9 @@ TEST_F(AvmExecutionTests, opCallOpcodes)
     std::vector<FF> returndata = {};
 
     // Generate Hint for call operation
-    ExecutionHints execution_hints = {};
-    execution_hints.returndata_hints.push_back({ 9, 8 }); // Return data
+    ExecutionHints execution_hints;
+    execution_hints.externalcall_hints.push_back(
+        { .success = 1, .return_data = { 9, 8 }, .l2_gas_used = 0, .da_gas_used = 0 });
 
     auto trace = Execution::gen_trace(instructions, returndata, calldata, public_inputs_vec, execution_hints);
     EXPECT_EQ(returndata, std::vector<FF>({ 9, 8, 1 })); // The 1 represents the success

barretenberg/cpp/src/barretenberg/vm/tests/avm_kernel.test.cpp

@@ -774,8 +774,7 @@ TEST_F(AvmKernelOutputPositiveTests, kernelSload)
     auto slot = 12345;
 
     // Provide a hint for sload value slot
-    ExecutionHints execution_hints;
-    execution_hints.side_effect_hints[0] = FF(value); // side effect counter -> value
+    ExecutionHints execution_hints({ { 0, value } }, {}, {}, {}, {}, {});
 
     auto apply_opcodes = [=](AvmTraceBuilder& trace_builder) {
         trace_builder.op_set(0, static_cast<uint128_t>(slot), slot_offset, AvmMemoryTag::FF);
@@ -849,8 +848,7 @@ TEST_F(AvmKernelOutputPositiveTests, kernelNoteHashExists)
     uint32_t metadata_offset = 420;
     auto exists = 1;
 
-    ExecutionHints execution_hints = {};
-    execution_hints.side_effect_hints[0] = exists; // side effect counter -> value
+    ExecutionHints execution_hints({ { 0, exists } }, {}, {}, {}, {}, {});
 
     auto apply_opcodes = [=](AvmTraceBuilder& trace_builder) {
         trace_builder.op_set(0, static_cast<uint128_t>(value), value_offset, AvmMemoryTag::FF);
@@ -888,8 +886,7 @@ TEST_F(AvmKernelOutputPositiveTests, kernelNullifierExists)
     uint32_t metadata_offset = 420;
     auto exists = 1;
 
-    ExecutionHints execution_hints = {};
-    execution_hints.side_effect_hints[0] = exists; // side effect counter -> value
+    ExecutionHints execution_hints({ { 0, exists } }, {}, {}, {}, {}, {});
 
     auto apply_opcodes = [=](AvmTraceBuilder& trace_builder) {
         trace_builder.op_set(0, static_cast<uint128_t>(value), value_offset, AvmMemoryTag::FF);
@@ -967,8 +964,7 @@ TEST_F(AvmKernelOutputPositiveTests, kernelL1ToL2MsgExists)
     auto exists = 1;
 
     // Create an execution hints object with the result of the operation
-    ExecutionHints execution_hints = {};
-    execution_hints.side_effect_hints[0] = exists;
+    ExecutionHints execution_hints({ { 0, exists } }, {}, {}, {}, {}, {});
 
     auto apply_opcodes = [=](AvmTraceBuilder& trace_builder) {
         trace_builder.op_set(0, static_cast<uint128_t>(value), value_offset, AvmMemoryTag::FF);

yarn-project/bb-prover/src/bb/execute.ts

@@ -319,7 +319,7 @@ export async function generateAvmProof(
       return { status: BB_RESULT.FAILURE, reason: `Could not write publicInputs at ${publicInputsPath}` };
     }
 
-    await fs.writeFile(avmHintsPath, input.avmHints.toBufferVM());
+    await fs.writeFile(avmHintsPath, input.avmHints.toBuffer());
     if (!filePresent(avmHintsPath)) {
       return { status: BB_RESULT.FAILURE, reason: `Could not write avmHints at ${avmHintsPath}` };
     }

yarn-project/circuits.js/src/structs/avm/avm.test.ts

@@ -1,49 +1,14 @@
 import { randomInt } from '@aztec/foundation/crypto';
 
-import { makeAvmCircuitInputs, makeAvmExecutionHints, makeAvmHint } from '../../tests/factories.js';
-import { AvmCircuitInputs, AvmExecutionHints, AvmHint } from './avm.js';
+import { makeAvmCircuitInputs } from '../../tests/factories.js';
+import { AvmCircuitInputs } from './avm.js';
 
 describe('Avm circuit inputs', () => {
-  describe('AvmHint', () => {
-    let avmHint: AvmHint;
-
-    beforeAll(() => {
-      avmHint = makeAvmHint(randomInt(1000));
-    });
-
-    it(`serializes to buffer and deserializes it back`, () => {
-      const buffer = avmHint.toBuffer();
-      const res = AvmHint.fromBuffer(buffer);
-      expect(res).toEqual(avmHint);
-      expect(res.isEmpty()).toBe(false);
-    });
-  });
-  describe('AvmExecutionHints', () => {
-    let avmExecutionHints: AvmExecutionHints;
-
-    beforeAll(() => {
-      avmExecutionHints = makeAvmExecutionHints(randomInt(1000));
-    });
-
-    it(`serializes to buffer and deserializes it back`, () => {
-      const buffer = avmExecutionHints.toBuffer();
-      const res = AvmExecutionHints.fromBuffer(buffer);
-      expect(res).toEqual(avmExecutionHints);
-      expect(res.isEmpty()).toBe(false);
-    });
-  });
-  describe('AvmCircuitInputs', () => {
-    let avmCircuitInputs: AvmCircuitInputs;
-
-    beforeAll(() => {
-      avmCircuitInputs = makeAvmCircuitInputs(randomInt(2000));
-    });
-
-    it(`serializes to buffer and deserializes it back`, () => {
-      const buffer = avmCircuitInputs.toBuffer();
-      const res = AvmCircuitInputs.fromBuffer(buffer);
-      expect(res).toEqual(avmCircuitInputs);
-      expect(res.isEmpty()).toBe(false);
-    });
+  it(`serializes to buffer and deserializes it back`, () => {
+    const avmCircuitInputs = makeAvmCircuitInputs(randomInt(2000));
+    const buffer = avmCircuitInputs.toBuffer();
+    const res = AvmCircuitInputs.fromBuffer(buffer);
+    expect(res).toEqual(avmCircuitInputs);
+    expect(res.isEmpty()).toBe(false);
   });
 });