LIVE-2160 BTC Support very large UTXOs (#1926)

* fix change output greater than MAX_SAFE_INT in js

* fix bch bot

* fix btc transaction serialization

* add unit test for huge utxo as output

* increase fees for qtum

* empty commit

* add varuint-bitcoin into dependencies.md

dependencies.md

@@ -105,3 +105,4 @@ yarn upgrade-interactive -i --latest
 |winston                 | logs                           | monthly                          |
 |xstate                  | generic helper for React       | **TBD why it's needed.**         |
 |zcash-bitcore-lib       | Bitcoin coin integration       | monthly                          |
+|varuint-bitcoin         | Bitcoin coin integration       | monthly                          |

package.json

@@ -133,6 +133,7 @@
     "stellar-sdk": "^10.0.1",
     "superstruct": "^0.14.2",
     "triple-beam": "^1.3.0",
+    "varuint-bitcoin": "1.1.2",
     "winston": "^3.4.0",
     "xstate": "^4.28.1"
   },

src/__tests__/families/bitcoin/wallet-btc/xpub.txs.dogecoin.test.ts

@@ -0,0 +1,96 @@
+import coininfo from "coininfo";
+import BigNumber from "bignumber.js";
+import { DerivationModes } from "../../../../families/bitcoin/wallet-btc/types";
+import Xpub from "../../../../families/bitcoin/wallet-btc/xpub";
+import Doge from "../../../../families/bitcoin/wallet-btc/crypto/doge";
+import BitcoinLikeExplorer from "../../../../families/bitcoin/wallet-btc/explorer";
+import BitcoinLikeStorage from "../../../../families/bitcoin/wallet-btc/storage";
+import { Merge } from "../../../../families/bitcoin/wallet-btc/pickingstrategies/Merge";
+import BitcoinLikeWallet from "../../../../families/bitcoin/wallet-btc/wallet";
+import MockBtc from "../../../../mock/Btc";
+
+describe("testing dogecoin transactions", () => {
+  const wallet = new BitcoinLikeWallet();
+  const explorer = new BitcoinLikeExplorer({
+    explorerURI: "https://explorers.api.vault.ledger.com/blockchain/v3/doge",
+    explorerVersion: "v3",
+    disableBatchSize: true,
+  });
+
+  const network = coininfo.dogecoin.main.toBitcoinJS();
+  const crypto = new Doge({ network });
+
+  const storage = new BitcoinLikeStorage();
+  const xpub = new Xpub({
+    storage,
+    explorer,
+    crypto,
+    xpub: "dgub8rBf7BYsf5YoMezYuPaEhc2tsr7sQA2v2xNCj4mt1czF1m4hRiBdjYeAq5xDVQhN5HqYQnxv2DwyfmDvp1QEfmi44b8uynPL45KXQJrsoi8",
+    derivationMode: DerivationModes.LEGACY,
+  });
+  it("testing dogecoin transactions with huge amount", async () => {
+    const utxoPickingStrategy = new Merge(xpub.crypto, xpub.derivationMode, []);
+    const changeAddress = await xpub.getNewAddress(1, 0);
+    xpub.storage.appendTxs([
+      {
+        id: "c4ee70c30b9c5c5fed60c37ce86046156af3623f32aa5be94556b35dcf0af147",
+        inputs: [],
+        outputs: [
+          {
+            output_index: 0,
+            value: "500000000000000000", // huge utxo
+            address: "mwXTtHo8Yy3aNKUUZLkBDrTcKT9qG9TqLb",
+            output_hash:
+              "c4ee70c30b9c5c5fed60c37ce86046156af3623f32aa5be94556b35dcf0af147",
+            block_height: 1,
+            rbf: false,
+          },
+          {
+            output_index: 1,
+            value: "0",
+            address: "<unknown>",
+            output_hash:
+              "c4ee70c30b9c5c5fed60c37ce86046156af3623f32aa5be94556b35dcf0af147",
+            block_height: 1,
+            rbf: false,
+          },
+        ],
+        block: {
+          hash: "73c565a6f226978df23480e440b27eb02f307855f50aa3bc72ebb586938f23e0",
+          height: 1,
+          time: "2021-07-28T13:34:17Z",
+        },
+        account: 0,
+        index: 0,
+        address: "mwXTtHo8Yy3aNKUUZLkBDrTcKT9qG9TqLb",
+        received_at: "2021-07-28T13:34:17Z",
+      },
+    ]);
+
+    const txInfo = await xpub.buildTx({
+      destAddress: "D9fSjc6zAyjdRgSfbfMLv5z5FpuacvguUi",
+      amount: new BigNumber(200000000000000000),
+      feePerByte: 100,
+      changeAddress,
+      utxoPickingStrategy,
+      sequence: 0,
+    });
+    const account = await wallet.generateAccount({
+      xpub: xpub.xpub,
+      path: "44'/0'",
+      index: 0,
+      currency: "dogecoin",
+      network: "mainnet",
+      derivationMode: DerivationModes.LEGACY,
+      explorer: "ledgerv3",
+      explorerURI: "https://explorers.api.vault.ledger.com/blockchain/v3/doge",
+      storage: "mock",
+      storageParams: [],
+    });
+    await wallet.signAccountTx({
+      btc: new MockBtc(),
+      fromAccount: account,
+      txInfo,
+    });
+  }, 100000);
+});

src/families/bitcoin/getAccountNetworkInfo.ts

@@ -47,10 +47,12 @@ export async function getAccountNetworkInfo(
   }
   // Fix fees if suggested fee is too low, this is only for viacoin/decred because the fees backend endpoint is broken
   if (
-    (account.currency.id === "viacoin" || account.currency.id === "decred") &&
+    (account.currency.id === "viacoin" ||
+      account.currency.id === "decred" ||
+      account.currency.id === "qtum") &&
     feesPerByte[2].toNumber() < Math.ceil(relayFee * 100000)
   ) {
-    feesPerByte[2] = new BigNumber(Math.ceil(relayFee * 100000)).plus(1);
+    feesPerByte[2] = new BigNumber(Math.ceil(relayFee * 100000)).plus(2);
     feesPerByte[1] = feesPerByte[2].plus(1);
     if (feesPerByte[1].plus(1).gt(feesPerByte[0])) {
       feesPerByte[0] = feesPerByte[1].plus(1);

src/families/bitcoin/specs.ts

@@ -10,11 +10,7 @@ import { pickSiblings } from "../../bot/specs";
 import { bitcoinPickingStrategy } from "./types";
 import type { MutationSpec, AppSpec } from "../../bot/types";
 import { LowerThanMinimumRelayFee } from "../../errors";
-import {
-  getMinRelayFee,
-  getUTXOStatus,
-  bchToCashaddrAddressWithoutPrefix,
-} from "./logic";
+import { getMinRelayFee, getUTXOStatus } from "./logic";
 import { DeviceModelId } from "@ledgerhq/devices";
 type Arg = Partial<{
   minimalAmount: BigNumber;
@@ -316,6 +312,9 @@ const bitcoinGold: AppSpec<Transaction> = {
   mutations: bitcoinLikeMutations(),
 };
 
+const bchToCashaddrAddressWithoutPrefix = (recipient) =>
+  bchaddrjs.toCashAddress(recipient).split(":")[1];
+
 const bitcoinCash: AppSpec<Transaction> = {
   name: "Bitcoin Cash",
   currency: getCryptoCurrencyById("bitcoin_cash"),

src/families/bitcoin/wallet-btc/utils.ts

@@ -7,6 +7,7 @@ import { Currency, ICrypto } from "./crypto/types";
 import cryptoFactory from "./crypto/factory";
 import { fallbackValidateAddress } from "./crypto/base";
 import { UnsupportedDerivation } from "../../../errors";
+import varuint from "varuint-bitcoin";
 
 export function parseHexString(str: any) {
   const result: Array<number> = [];
@@ -261,3 +262,10 @@ export function isTaprootAddress(address: string, currency?: Currency) {
     return false;
   }
 }
+
+export function writeVarInt(buffer: Buffer, i: number, offset: number) {
+  // refer to https://github.com/bitcoinjs/bitcoinjs-lib/blob/1f44f722d30cd14a1861c8546e6b455f73862c1e/src/bufferutils.js#L78
+  varuint.encode(i, buffer, offset);
+  offset += varuint.encode.bytes;
+  return offset;
+}

src/families/bitcoin/wallet-btc/wallet.ts

@@ -2,15 +2,10 @@
 // @ts-ignore
 import { flatten } from "lodash";
 import BigNumber from "bignumber.js";
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore
-import { BufferWriter } from "bitcoinjs-lib/src/bufferutils";
-
 import Btc from "@ledgerhq/hw-app-btc";
 import { log } from "@ledgerhq/logs";
 import { Transaction } from "@ledgerhq/hw-app-btc/lib/types";
 import { Currency } from "./crypto/types";
-
 import { TransactionInfo, DerivationModes } from "./types";
 import { Account, SerializedAccount } from "./account";
 import Xpub from "./xpub";
@@ -246,16 +241,33 @@ class BitcoinLikeWallet {
       length += 2 * txInfo.outputs.length;
     }
     const buffer = Buffer.allocUnsafe(length);
-    const bufferWriter = new BufferWriter(buffer, 0);
-    bufferWriter.writeVarInt(txInfo.outputs.length);
+    let bufferOffset = 0;
+    bufferOffset = utils.writeVarInt(
+      buffer,
+      txInfo.outputs.length,
+      bufferOffset
+    );
     txInfo.outputs.forEach((txOut) => {
-      // xpub splits output into smaller outputs than SAFE_MAX_INT anyway
-      bufferWriter.writeUInt64(txOut.value.toNumber());
+      // refer to https://github.com/bitcoinjs/bitcoinjs-lib/blob/59b21162a2c4645c64271ca004c7a3755a3d72fb/ts_src/bufferutils.ts#L26
+      buffer.writeUInt32LE(
+        txOut.value.modulo(new BigNumber(0x100000000)).toNumber(),
+        bufferOffset
+      );
+      buffer.writeUInt32LE(
+        txOut.value.dividedToIntegerBy(new BigNumber(0x100000000)).toNumber(),
+        bufferOffset + 4
+      );
+      bufferOffset += 8;
       if (additionals && additionals.includes("decred")) {
-        bufferWriter.writeVarInt(0);
-        bufferWriter.writeVarInt(0);
+        bufferOffset = utils.writeVarInt(buffer, 0, bufferOffset);
+        bufferOffset = utils.writeVarInt(buffer, 0, bufferOffset);
       }
-      bufferWriter.writeVarSlice(txOut.script);
+      bufferOffset = utils.writeVarInt(
+        buffer,
+        txOut.script.length,
+        bufferOffset
+      );
+      bufferOffset += txOut.script.copy(buffer, bufferOffset);
     });
     const outputScriptHex = buffer.toString("hex");
     const associatedKeysets = txInfo.associatedDerivations.map(