feat: note preprocessor (#7857)

docs/docs/reference/developer_references/smart_contract_reference/storage/private_state.md

@@ -269,9 +269,21 @@ When the `limit` is set to a non-zero value, the data oracle will return a maxim
 
 This setting enables us to skip the first `offset` notes. It's particularly useful for pagination.
 
+### `preprocessor: fn ([Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL]`
+
+Developers have the option to provide a custom preprocessor.
+This allows specific logic to be applied to notes that meet the criteria outlined above.
+The preprocessor takes the notes returned from the oracle and `preprocessor_args` as its parameters.
+
+An important distinction from the filter function described below is that preprocessor is applied first and unlike filter it is applied in an unconstrained context.
+
+### `preprocessor_args: PREPROCESSOR_ARGS`
+
+`preprocessor_args` provides a means to furnish additional data or context to the custom preprocessor.
+
 ### `filter: fn ([Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], FILTER_ARGS) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL]`
 
-Developers have the option to provide a custom filter. This allows specific logic to be applied to notes that meet the criteria outlined above. The filter takes the notes returned from the oracle and `filter_args` as its parameters.
+Just like preprocessor just applied in a constrained context (correct execution is proven) and applied after the preprocessor.
 
 ### `filter_args: FILTER_ARGS`
 

noir-projects/aztec-nr/aztec/src/note/note_getter/mod.nr

@@ -103,21 +103,29 @@ pub fn get_note<Note, let N: u32, let M: u32>(
     note
 }
 
-pub fn get_notes<Note, let N: u32, let M: u32, FILTER_ARGS>(
+pub fn get_notes<Note, let N: u32, let M: u32, PREPROCESSOR_ARGS, FILTER_ARGS>(
     context: &mut PrivateContext,
     storage_slot: Field,
-    options: NoteGetterOptions<Note, N, M, FILTER_ARGS>
+    options: NoteGetterOptions<Note, N, M, PREPROCESSOR_ARGS, FILTER_ARGS>
 ) -> BoundedVec<Note, MAX_NOTE_HASH_READ_REQUESTS_PER_CALL> where Note: NoteInterface<N, M> + Eq {
     let opt_notes = get_notes_internal(storage_slot, options);
 
     constrain_get_notes_internal(context, storage_slot, opt_notes, options)
 }
 
-fn constrain_get_notes_internal<Note, let N: u32, let M: u32, FILTER_ARGS>(
+unconstrained fn apply_preprocessor<Note, let N: u32, let M: u32, PREPROCESSOR_ARGS>(
+    notes: [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
+    preprocessor: fn([Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
+    preprocessor_args: PREPROCESSOR_ARGS
+) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL] {
+    preprocessor(notes, preprocessor_args)
+}
+
+fn constrain_get_notes_internal<Note, let N: u32, let M: u32, PREPROCESSOR_ARGS, FILTER_ARGS>(
     context: &mut PrivateContext,
     storage_slot: Field,
     opt_notes: [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
-    options: NoteGetterOptions<Note, N, M, FILTER_ARGS>
+    options: NoteGetterOptions<Note, N, M, PREPROCESSOR_ARGS, FILTER_ARGS>
 ) -> BoundedVec<Note, MAX_NOTE_HASH_READ_REQUESTS_PER_CALL> where Note: NoteInterface<N, M> + Eq {
     // The filter is applied first to avoid pushing note read requests for notes we're not interested in. Note that
     // while the filter function can technically mutate the contents of the notes (as opposed to simply removing some),
@@ -181,9 +189,9 @@ unconstrained fn get_note_internal<Note, let N: u32, let M: u32>(storage_slot: F
     )[0].unwrap() // Notice: we don't allow dummies to be returned from get_note (singular).
 }
 
-unconstrained fn get_notes_internal<Note, let N: u32, let M: u32, FILTER_ARGS>(
+unconstrained fn get_notes_internal<Note, let N: u32, let M: u32, PREPROCESSOR_ARGS, FILTER_ARGS>(
     storage_slot: Field,
-    options: NoteGetterOptions<Note, N, M, FILTER_ARGS>
+    options: NoteGetterOptions<Note, N, M, PREPROCESSOR_ARGS, FILTER_ARGS>
 ) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL] where Note: NoteInterface<N, M> {
     // This function simply performs some transformations from NoteGetterOptions into the types required by the oracle.
 
@@ -192,7 +200,7 @@ unconstrained fn get_notes_internal<Note, let N: u32, let M: u32, FILTER_ARGS>(
     let placeholder_fields = [0; GET_NOTES_ORACLE_RETURN_LENGTH];
     let placeholder_note_length = [0; N];
 
-    oracle::notes::get_notes(
+    let opt_notes = oracle::notes::get_notes(
         storage_slot,
         num_selects,
         select_by_indexes,
@@ -210,7 +218,9 @@ unconstrained fn get_notes_internal<Note, let N: u32, let M: u32, FILTER_ARGS>(
         placeholder_opt_notes,
         placeholder_fields,
         placeholder_note_length
-    )
+    );
+
+    apply_preprocessor(opt_notes, options.preprocessor, options.preprocessor_args)
 }
 
 unconstrained pub fn view_notes<Note, let N: u32, let M: u32>(

noir-projects/aztec-nr/aztec/src/note/note_getter_options.nr

@@ -78,46 +78,77 @@ fn return_all_notes<Note, let N: u32>(
 }
 
 // docs:start:NoteGetterOptions
-struct NoteGetterOptions<Note, let N: u32, let M: u32, FILTER_ARGS> {
+struct NoteGetterOptions<Note, let N: u32, let M: u32, PREPROCESSOR_ARGS, FILTER_ARGS> {
     selects: BoundedVec<Option<Select>, N>,
     sorts: BoundedVec<Option<Sort>, N>,
     limit: u32,
     offset: u32,
+    // Preprocessor and filter functions are used to filter notes. The preprocessor is applied before the filter and
+    // unlike filter it is applied in an unconstrained context.
+    preprocessor: fn ([Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
+    preprocessor_args: PREPROCESSOR_ARGS,
     filter: fn ([Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], FILTER_ARGS) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
     filter_args: FILTER_ARGS,
     status: u8,
 }
 // docs:end:NoteGetterOptions
 
-// When retrieving notes using the NoteGetterOptions, the configurations are applied in a specific sequence to ensure precise and controlled data retrieval.
+// When retrieving notes using the NoteGetterOptions, the configurations are applied in a specific sequence to ensure
+// precise and controlled data retrieval.
 // The database-level configurations are applied first:
-// `selects` to specify fields, `sorts` to establish sorting criteria, `offset` to skip items, and `limit` to cap the result size.
-// And finally, a custom filter to refine the outcome further.
-impl<Note, let N: u32, let M: u32, FILTER_ARGS> NoteGetterOptions<Note, N, M, FILTER_ARGS> {
+// `selects` to specify fields, `sorts` to establish sorting criteria, `offset` to skip items, and `limit` to cap
+// the result size.
+// And finally, a custom preprocessor and filter to refine the outcome further.
+impl<Note, let N: u32, let M: u32, PREPROCESSOR_ARGS, FILTER_ARGS> NoteGetterOptions<Note, N, M, PREPROCESSOR_ARGS, FILTER_ARGS> {
     // This function initializes a NoteGetterOptions that simply returns the maximum number of notes allowed in a call.
-    pub fn new() -> NoteGetterOptions<Note, N, M, Field> where Note: NoteInterface<N, M> {
+    pub fn new() -> NoteGetterOptions<Note, N, M, Field, Field> where Note: NoteInterface<N, M> {
         NoteGetterOptions {
             selects: BoundedVec::new(),
             sorts: BoundedVec::new(),
             limit: MAX_NOTE_HASH_READ_REQUESTS_PER_CALL as u32,
             offset: 0,
+            preprocessor: return_all_notes,
+            preprocessor_args: 0,
             filter: return_all_notes,
             filter_args: 0,
             status: NoteStatus.ACTIVE
         }
     }
 
-    // This function initializes a NoteGetterOptions with a filter, which takes the notes returned from the database and filter_args as its parameters.
+    // This function initializes a NoteGetterOptions with a preprocessor, which takes the notes returned from
+    // the database and preprocessor_args as its parameters.
+    // `preprocessor_args` allows you to provide additional data or context to the custom preprocessor.
+    pub fn with_preprocessor(
+        preprocessor: fn([Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], PREPROCESSOR_ARGS) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
+        preprocessor_args: PREPROCESSOR_ARGS
+    ) -> NoteGetterOptions<Note, N, M, PREPROCESSOR_ARGS, Field> where Note: NoteInterface<N, M> {
+        NoteGetterOptions {
+            selects: BoundedVec::new(),
+            sorts: BoundedVec::new(),
+            limit: MAX_NOTE_HASH_READ_REQUESTS_PER_CALL as u32,
+            offset: 0,
+            preprocessor,
+            preprocessor_args,
+            filter: return_all_notes,
+            filter_args: 0,
+            status: NoteStatus.ACTIVE
+        }
+    }
+
+    // This function initializes a NoteGetterOptions with a filter, which takes
+    // the notes returned from the database and filter_args as its parameters.
     // `filter_args` allows you to provide additional data or context to the custom filter.
     pub fn with_filter(
         filter: fn([Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL], FILTER_ARGS) -> [Option<Note>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
         filter_args: FILTER_ARGS
-    ) -> Self where Note: NoteInterface<N, M> {
+    ) -> NoteGetterOptions<Note, N, M, Field, FILTER_ARGS> where Note: NoteInterface<N, M> {
         NoteGetterOptions {
             selects: BoundedVec::new(),
             sorts: BoundedVec::new(),
             limit: MAX_NOTE_HASH_READ_REQUESTS_PER_CALL as u32,
             offset: 0,
+            preprocessor: return_all_notes,
+            preprocessor_args: 0,
             filter,
             filter_args,
             status: NoteStatus.ACTIVE

noir-projects/aztec-nr/aztec/src/state_vars/private_set.nr

@@ -41,9 +41,9 @@ impl<Note, let N: u32, let M: u32> PrivateSet<Note, &mut PrivateContext> where N
     }
     // docs:end:insert
 
-    pub fn pop_notes<FILTER_ARGS>(
+    pub fn pop_notes<PREPROCESSOR_ARGS, FILTER_ARGS>(
         self,
-        options: NoteGetterOptions<Note, N, M, FILTER_ARGS>
+        options: NoteGetterOptions<Note, N, M, PREPROCESSOR_ARGS, FILTER_ARGS>
     ) -> BoundedVec<Note, MAX_NOTE_HASH_READ_REQUESTS_PER_CALL> {
         let notes = get_notes(self.context, self.storage_slot, options);
         // We iterate in a range 0..options.limit instead of 0..notes.len() because options.limit is known at compile
@@ -73,9 +73,9 @@ impl<Note, let N: u32, let M: u32> PrivateSet<Note, &mut PrivateContext> where N
 
     /// Note that if you later on remove the note it's much better to use `pop_notes` as `pop_notes` results
     /// in significantly less constrains due to avoiding 1 read request check.
-    pub fn get_notes<FILTER_ARGS>(
+    pub fn get_notes<PREPROCESSOR_ARGS, FILTER_ARGS>(
         self,
-        options: NoteGetterOptions<Note, N, M, FILTER_ARGS>
+        options: NoteGetterOptions<Note, N, M, PREPROCESSOR_ARGS, FILTER_ARGS>
     ) -> BoundedVec<Note, MAX_NOTE_HASH_READ_REQUESTS_PER_CALL> {
         get_notes(self.context, self.storage_slot, options)
     }

noir-projects/aztec-nr/aztec/src/utils/point.nr

@@ -9,6 +9,8 @@ global BN254_FR_MODULUS_DIV_2: Field = 10944121435919637611123202872628637544274
 /// We don't serialize the point at infinity flag because this function is used in situations where we do not want
 /// to waste the extra byte (encrypted log).
 pub fn point_to_bytes(pk: Point) -> [u8; 32] {
+    // Note that there is 1 more free bit in the 32 bytes (254 bits currently occupied by the x coordinate, 1 bit for
+    // the "sign") so it's possible to use that last bit as an "is_infinite" flag if desired in the future.
     assert(!pk.is_infinite, "Cannot serialize point at infinity as bytes.");
 
     let mut result = pk.x.to_be_bytes(32);

noir-projects/aztec-nr/value-note/src/utils.nr

@@ -5,7 +5,7 @@ use crate::{filter::filter_notes_min_sum, value_note::{ValueNote, VALUE_NOTE_LEN
 
 // Sort the note values (0th field) in descending order.
 // Pick the fewest notes whose sum is equal to or greater than `amount`.
-pub fn create_note_getter_options_for_decreasing_balance(amount: Field) -> NoteGetterOptions<ValueNote, VALUE_NOTE_LEN, VALUE_NOTE_BYTES_LEN, Field> {
+pub fn create_note_getter_options_for_decreasing_balance(amount: Field) -> NoteGetterOptions<ValueNote, VALUE_NOTE_LEN, VALUE_NOTE_BYTES_LEN, Field, Field> {
     NoteGetterOptions::with_filter(filter_notes_min_sum, amount).sort(ValueNote::properties().value, SortOrder.DESC)
 }
 

noir-projects/noir-contracts/contracts/docs_example_contract/src/options.nr

@@ -10,7 +10,7 @@ use dep::aztec::note::note_getter_options::{Sort, SortOrder};
 pub fn create_points_card_getter_options(
     points: Field,
     offset: u32
-) -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, Field> {
+) -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, Field, Field> {
     let mut options = NoteGetterOptions::new();
     options.select(CardNote::properties().points, points, Option::none()).sort(CardNote::properties().points, SortOrder.DESC).set_offset(offset)
 }
@@ -21,7 +21,7 @@ pub fn create_exact_card_getter_options(
     points: u8,
     secret: Field,
     account_npk_m_hash: Field
-) -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, Field> {
+) -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, Field, Field> {
     let mut options = NoteGetterOptions::new();
     options.select(CardNote::properties().points, points as Field, Option::none()).select(CardNote::properties().randomness, secret, Option::none()).select(
         CardNote::properties().npk_m_hash,
@@ -49,13 +49,13 @@ pub fn filter_min_points(
 // docs:end:state_vars-OptionFilter
 
 // docs:start:state_vars-NoteGetterOptionsFilter
-pub fn create_cards_with_min_points_getter_options(min_points: u8) -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, u8> {
+pub fn create_cards_with_min_points_getter_options(min_points: u8) -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, Field, u8> {
     NoteGetterOptions::with_filter(filter_min_points, min_points).sort(CardNote::properties().points, SortOrder.ASC)
 }
 // docs:end:state_vars-NoteGetterOptionsFilter
 
 // docs:start:state_vars-NoteGetterOptionsPickOne
-pub fn create_largest_card_getter_options() -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, Field> {
+pub fn create_largest_card_getter_options() -> NoteGetterOptions<CardNote, CARD_NOTE_LEN, CARD_NOTE_BYTES_LEN, Field, Field> {
     let mut options = NoteGetterOptions::new();
     options.sort(CardNote::properties().points, SortOrder.DESC).set_limit(1)
 }

noir-projects/noir-contracts/contracts/token_contract/src/types/balances_map.nr

@@ -107,7 +107,12 @@ impl<T> BalancesMap<T, &mut PrivateContext> {
         target_amount: U128,
         max_notes: u32
     ) -> U128 where T: NoteInterface<T_SERIALIZED_LEN, T_SERIALIZED_BYTES_LEN> + OwnedNote + Eq {
-        let options = NoteGetterOptions::with_filter(filter_notes_min_sum, target_amount).set_limit(max_notes);
+        // We are using a preprocessor here (filter applied in an unconstrained context) instead of a filter because
+        // we do not need to prove correct execution of the preprocessor.
+        // Because the `min_sum` notes is not constrained, users could choose to e.g. not call it. However, all this
+        // might result in is simply higher DA costs due to more nullifiers being emitted. Since we don't care
+        // about proving optimal note usage, we can save these constraints and make the circuit smaller.
+        let options = NoteGetterOptions::with_preprocessor(preprocess_notes_min_sum, target_amount).set_limit(max_notes);
         let notes = self.map.at(owner).pop_notes(options);
 
         let mut subtracted = U128::from_integer(0);
@@ -124,10 +129,10 @@ impl<T> BalancesMap<T, &mut PrivateContext> {
 
 // Computes the partial sum of the notes array, stopping once 'min_sum' is reached. This can be used to minimize the
 // number of notes read that add to some value, e.g. when transferring some amount of tokens.
-// The filter does not check if total sum is larger or equal to 'min_sum' - all it does is remove extra notes if it does
-// reach that value.
-// Note that proper usage of this filter requires for notes to be sorted in descending order.
-pub fn filter_notes_min_sum<T, T_SERIALIZED_LEN, T_SERIALIZED_BYTES_LEN>(
+// The preprocessor (a filter applied in an unconstrained context) does not check if total sum is larger or equal to
+// 'min_sum' - all it does is remove extra notes if it does reach that value.
+// Note that proper usage of this preprocessor requires for notes to be sorted in descending order.
+pub fn preprocess_notes_min_sum<T, T_SERIALIZED_LEN, T_SERIALIZED_BYTES_LEN>(
     notes: [Option<T>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL],
     min_sum: U128
 ) -> [Option<T>; MAX_NOTE_HASH_READ_REQUESTS_PER_CALL] where T: NoteInterface<T_SERIALIZED_LEN, T_SERIALIZED_BYTES_LEN> + OwnedNote {