merge queue: embarking main (a835270) and #6098 together

zebra-chain/src/transaction/builder.rs

@@ -15,6 +15,7 @@ impl Transaction {
         network: Network,
         height: Height,
         outputs: impl IntoIterator<Item = (Amount<NonNegative>, transparent::Script)>,
+        extra_coinbase_data: Vec<u8>,
     ) -> Transaction {
         // # Consensus
         //
@@ -36,13 +37,17 @@ impl Transaction {
         //
         // > A coinbase transaction script MUST have length in {2 .. 100} bytes.
         //
-        // Zebra does not add any extra coinbase data.
+        // Zebra adds extra coinbase data if configured to do so.
         //
         // Since we're not using a lock time, any sequence number is valid here.
         // See `Transaction::lock_time()` for the relevant consensus rules.
         //
         // <https://zips.z.cash/protocol/protocol.pdf#txnconsensus>
-        let inputs = vec![transparent::Input::new_coinbase(height, None, None)];
+        let inputs = vec![transparent::Input::new_coinbase(
+            height,
+            Some(extra_coinbase_data),
+            None,
+        )];
 
         // > The block subsidy is composed of a miner subsidy and a series of funding streams.
         //
@@ -108,17 +113,23 @@ impl Transaction {
         height: Height,
         outputs: impl IntoIterator<Item = (Amount<NonNegative>, transparent::Script)>,
         like_zcashd: bool,
+        extra_coinbase_data: Vec<u8>,
     ) -> Transaction {
-        // `zcashd` includes an extra byte after the coinbase height in the coinbase data,
-        // and a sequence number of u32::MAX.
         let mut extra_data = None;
         let mut sequence = None;
 
+        // `zcashd` includes an extra byte after the coinbase height in the coinbase data,
+        // and a sequence number of u32::MAX.
         if like_zcashd {
             extra_data = Some(vec![0x00]);
             sequence = Some(u32::MAX);
         }
 
+        // Override like_zcashd if extra_coinbase_data was supplied
+        if !extra_coinbase_data.is_empty() {
+            extra_data = Some(extra_coinbase_data);
+        }
+
         // # Consensus
         //
         // See the other consensus rules above in new_v5_coinbase().

zebra-chain/src/transparent.rs

@@ -1,5 +1,15 @@
 //! Transparent-related (Bitcoin-inherited) functionality.
 
+use std::{collections::HashMap, fmt, iter};
+
+use crate::{
+    amount::{Amount, NonNegative},
+    block,
+    parameters::Network,
+    primitives::zcash_primitives,
+    transaction,
+};
+
 mod address;
 mod keys;
 mod opcodes;
@@ -9,7 +19,7 @@ mod utxo;
 
 pub use address::Address;
 pub use script::Script;
-pub use serialize::GENESIS_COINBASE_DATA;
+pub use serialize::{GENESIS_COINBASE_DATA, MAX_COINBASE_DATA_LEN, MAX_COINBASE_HEIGHT_DATA_LEN};
 pub use utxo::{
     new_ordered_outputs, new_outputs, outputs_from_utxos, utxos_from_ordered_utxos,
     CoinbaseSpendRestriction, OrderedUtxo, Utxo,
@@ -20,24 +30,14 @@ pub use utxo::{
     new_ordered_outputs_with_height, new_outputs_with_height, new_transaction_ordered_outputs,
 };
 
-#[cfg(any(test, feature = "proptest-impl"))]
-use proptest_derive::Arbitrary;
-
 #[cfg(any(test, feature = "proptest-impl"))]
 mod arbitrary;
 
 #[cfg(test)]
 mod tests;
 
-use crate::{
-    amount::{Amount, NonNegative},
-    block,
-    parameters::Network,
-    primitives::zcash_primitives,
-    transaction,
-};
-
-use std::{collections::HashMap, fmt, iter};
+#[cfg(any(test, feature = "proptest-impl"))]
+use proptest_derive::Arbitrary;
 
 /// The maturity threshold for transparent coinbase outputs.
 ///
@@ -48,16 +48,28 @@ use std::{collections::HashMap, fmt, iter};
 /// [7.1](https://zips.z.cash/protocol/nu5.pdf#txnencodingandconsensus)
 pub const MIN_TRANSPARENT_COINBASE_MATURITY: u32 = 100;
 
+/// Extra coinbase data that identifies some coinbase transactions generated by Zebra.
+/// <https://emojipedia.org/zebra/>
+//
+// # Note
+//
+// rust-analyzer will crash in some editors when moving over an actual Zebra emoji,
+// so we encode it here. This is a known issue in emacs-lsp and other lsp implementations:
+// - https://github.com/rust-lang/rust-analyzer/issues/9121
+// - https://github.com/emacs-lsp/lsp-mode/issues/2080
+// - https://github.com/rust-lang/rust-analyzer/issues/13709
+pub const EXTRA_ZEBRA_COINBASE_DATA: &str = "z\u{1F993}";
+
 /// Arbitrary data inserted by miners into a coinbase transaction.
+//
+// TODO: rename to ExtraCoinbaseData, because height is also part of the coinbase data?
 #[derive(Clone, Eq, PartialEq)]
 #[cfg_attr(any(test, feature = "proptest-impl"), derive(Serialize))]
 pub struct CoinbaseData(
     /// Invariant: this vec, together with the coinbase height, must be less than
     /// 100 bytes. We enforce this by only constructing CoinbaseData fields by
-    /// parsing blocks with 100-byte data fields. When we implement block
-    /// creation, we should provide a constructor for the coinbase data field
-    /// that restricts it to 95 = 100 -1 -4 bytes (safe for any block height up
-    /// to 500_000_000).
+    /// parsing blocks with 100-byte data fields, and checking newly created
+    /// CoinbaseData lengths in the transaction builder.
     pub(super) Vec<u8>,
 );
 
@@ -182,24 +194,54 @@ impl fmt::Display for Input {
 
 impl Input {
     /// Returns a new coinbase input for `height` with optional `data` and `sequence`.
+    ///
+    /// # Consensus
+    ///
+    /// The combined serialized size of `height` and `data` can be at most 100 bytes.
+    ///
+    /// > A coinbase transaction script MUST have length in {2 .. 100} bytes.
+    ///
+    /// <https://zips.z.cash/protocol/protocol.pdf#txnconsensus>
+    ///
+    /// # Panics
+    ///
+    /// If the coinbase data is greater than [`MAX_COINBASE_DATA_LEN`].
     #[cfg(feature = "getblocktemplate-rpcs")]
     pub fn new_coinbase(
         height: block::Height,
         data: Option<Vec<u8>>,
         sequence: Option<u32>,
     ) -> Input {
+        // "No extra coinbase data" is the default.
+        let data = data.unwrap_or_default();
+        let height_size = height.coinbase_zcash_serialized_size();
+
+        assert!(
+            data.len() + height_size <= MAX_COINBASE_DATA_LEN,
+            "invalid coinbase data: extra data {} bytes + height {height_size} bytes \
+             must be {} or less",
+            data.len(),
+            MAX_COINBASE_DATA_LEN,
+        );
+
         Input::Coinbase {
             height,
-
-            // "No extra coinbase data" is the default.
-            data: CoinbaseData(data.unwrap_or_default()),
+            data: CoinbaseData(data),
 
             // If the caller does not specify the sequence number,
             // use a sequence number that activates the LockTime.
             sequence: sequence.unwrap_or(0),
         }
     }
 
+    /// Returns the extra coinbase data in this input, if it is an [`Input::Coinbase`].
+    pub fn extra_coinbase_data(&self) -> Option<&CoinbaseData> {
+        match self {
+            Input::PrevOut { .. } => None,
+            Input::Coinbase { data, .. } => Some(data),
+        }
+    }
+
     /// Returns the input's sequence number.
     pub fn sequence(&self) -> u32 {
         match self {

zebra-chain/src/transparent/serialize.rs

@@ -5,9 +5,9 @@ use std::io;
 use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 
 use crate::{
-    block,
+    block::{self, Height},
     serialization::{
-        zcash_serialize_bytes, ReadZcashExt, SerializationError, ZcashDeserialize,
+        zcash_serialize_bytes, FakeWriter, ReadZcashExt, SerializationError, ZcashDeserialize,
         ZcashDeserializeInto, ZcashSerialize,
     },
     transaction,
@@ -26,6 +26,16 @@ use super::{CoinbaseData, Input, OutPoint, Output, Script};
 /// <https://zips.z.cash/protocol/protocol.pdf#txnconsensus>
 pub const MAX_COINBASE_DATA_LEN: usize = 100;
 
+/// The maximum length of the encoded coinbase height.
+///
+/// # Consensus
+///
+/// > The length of heightBytes MUST be in the range {1 .. 5}. Then the encoding is the length
+/// > of heightBytes encoded as one byte, followed by heightBytes itself.
+///
+/// <https://zips.z.cash/protocol/protocol.pdf#txnconsensus>
+pub const MAX_COINBASE_HEIGHT_DATA_LEN: usize = 6;
+
 /// The minimum length of the coinbase data.
 ///
 /// Includes the encoded coinbase height, if any.
@@ -100,7 +110,6 @@ impl ZcashDeserialize for OutPoint {
 pub(crate) fn parse_coinbase_height(
     mut data: Vec<u8>,
 ) -> Result<(block::Height, CoinbaseData), SerializationError> {
-    use block::Height;
     match (data.first(), data.len()) {
         // Blocks 1 through 16 inclusive encode block height with OP_N opcodes.
         (Some(op_n @ 0x51..=0x60), len) if len >= 1 => Ok((
@@ -226,6 +235,17 @@ pub(crate) fn write_coinbase_height<W: io::Write>(
     Ok(())
 }
 
+impl Height {
+    /// Get the size of `Height` when serialized into a coinbase input script.
+    pub fn coinbase_zcash_serialized_size(&self) -> usize {
+        let mut writer = FakeWriter(0);
+        let empty_data = CoinbaseData(Vec::new());
+
+        write_coinbase_height(*self, &empty_data, &mut writer).expect("writer should never fail");
+        writer.0
+    }
+}
+
 impl ZcashSerialize for Input {
     /// Serialize this transparent input.
     ///

zebra-rpc/src/methods.rs

@@ -262,6 +262,10 @@ where
     /// no matter what the estimated height or local clock is.
     debug_force_finished_sync: bool,
 
+    /// Test-only option that makes RPC responses more like `zcashd`.
+    #[allow(dead_code)]
+    debug_like_zcashd: bool,
+
     // Services
     //
     /// A handle to the mempool service.
@@ -301,6 +305,7 @@ where
         app_version: Version,
         network: Network,
         debug_force_finished_sync: bool,
+        debug_like_zcashd: bool,
         mempool: Buffer<Mempool, mempool::Request>,
         state: State,
         latest_chain_tip: Tip,
@@ -323,6 +328,7 @@ where
             app_version,
             network,
             debug_force_finished_sync,
+            debug_like_zcashd,
             mempool: mempool.clone(),
             state: state.clone(),
             latest_chain_tip: latest_chain_tip.clone(),
@@ -763,14 +769,14 @@ where
         use zebra_chain::block::MAX_BLOCK_BYTES;
 
         #[cfg(feature = "getblocktemplate-rpcs")]
-        /// Determines whether the output of this RPC is sorted like zcashd
-        const SHOULD_USE_ZCASHD_ORDER: bool = true;
+        // Determines whether the output of this RPC is sorted like zcashd
+        let should_use_zcashd_order = self.debug_like_zcashd;
 
         let mut mempool = self.mempool.clone();
 
         async move {
             #[cfg(feature = "getblocktemplate-rpcs")]
-            let request = if SHOULD_USE_ZCASHD_ORDER {
+            let request = if should_use_zcashd_order {
                 mempool::Request::FullTransactions
             } else {
                 mempool::Request::TransactionIds