diff --git a/app/check_tx.go b/app/check_tx.go index 6d67c52780..99941d97be 100644 --- a/app/check_tx.go +++ b/app/check_tx.go @@ -27,7 +27,7 @@ func (app *App) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx { if _, ok := msg.(*blobtypes.MsgPayForBlob); !ok { continue } - return sdkerrors.ResponseCheckTxWithEvents(blobtypes.ErrBloblessPFB, 0, 0, []abci.Event{}, false) + return sdkerrors.ResponseCheckTxWithEvents(blobtypes.ErrNoBlobs, 0, 0, []abci.Event{}, false) } // don't do anything special if we have a normal transaction return app.BaseApp.CheckTx(req) diff --git a/app/estimate_square_size.go b/app/estimate_square_size.go index ee9cf7af4b..eaf4d667c1 100644 --- a/app/estimate_square_size.go +++ b/app/estimate_square_size.go @@ -1,10 +1,12 @@ package app import ( + "encoding/binary" "math" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" coretypes "github.com/tendermint/tendermint/types" ) @@ -23,8 +25,7 @@ func estimateSquareSize(txs []parsedTx) (squareSize uint64, nonreserveStart int) if len(ptx.normalTx) != 0 { continue } - dataUsed := len(ptx.blobTx.Blobs[0].Data) - blobSharesUsed += shares.SparseSharesNeeded(uint32(dataUsed)) + blobSharesUsed += blobtypes.BlobTxSharesUsed(ptx.blobTx) } // assume that we have to add a lot of padding by simply doubling the number @@ -48,7 +49,8 @@ func estimateSquareSize(txs []parsedTx) (squareSize uint64, nonreserveStart int) // estimateTxShares estimates the number of shares used by transactions. func estimateTxShares(squareSize uint64, ptxs []parsedTx) int { - maxWTxOverhead := maxWrappedTxOverhead(squareSize) + maxWTxOverhead := maxIndexWrapperOverhead(squareSize) + maxIndexOverhead := maxIndexOverhead(squareSize) txbytes := 0 for _, pTx := range ptxs { if len(pTx.normalTx) != 0 { @@ -57,7 +59,7 @@ func estimateTxShares(squareSize uint64, ptxs []parsedTx) int { txbytes += txLen continue } - txLen := len(pTx.blobTx.Tx) + maxWTxOverhead + txLen := len(pTx.blobTx.Tx) + maxWTxOverhead + (maxIndexOverhead * len(pTx.blobTx.Blobs)) txLen += shares.DelimLen(uint64(txLen)) txbytes += txLen } @@ -70,7 +72,7 @@ func estimateTxShares(squareSize uint64, ptxs []parsedTx) int { // // TODO: make more efficient by only generating these numbers once or something // similar. This function alone can take up to 5ms. -func maxWrappedTxOverhead(squareSize uint64) int { +func maxIndexWrapperOverhead(squareSize uint64) int { maxTxLen := squareSize * squareSize * appconsts.ContinuationCompactShareContentSize wtx, err := coretypes.MarshalIndexWrapper( make([]byte, maxTxLen), @@ -81,3 +83,19 @@ func maxWrappedTxOverhead(squareSize uint64) int { } return len(wtx) - int(maxTxLen) } + +// maxIndexOverhead calculates the maximum amount of overhead in bytes that +// could occur by adding an index to an IndexWrapper. +func maxIndexOverhead(squareSize uint64) int { + maxShareIndex := squareSize * squareSize + maxIndexLen := binary.PutUvarint(make([]byte, binary.MaxVarintLen32), maxShareIndex) + wtx, err := coretypes.MarshalIndexWrapper(make([]byte, 1), uint32(maxShareIndex)) + if err != nil { + panic(err) + } + wtx2, err := coretypes.MarshalIndexWrapper(make([]byte, 1), uint32(maxShareIndex), uint32(maxShareIndex-1)) + if err != nil { + panic(err) + } + return len(wtx2) - len(wtx) + maxIndexLen +} diff --git a/app/estimate_square_size_test.go b/app/estimate_square_size_test.go index 4dbaa6910c..43c7d031ff 100644 --- a/app/estimate_square_size_test.go +++ b/app/estimate_square_size_test.go @@ -3,8 +3,12 @@ package app import ( "testing" + "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/celestia-app/testutil/blobfactory" + "github.com/celestiaorg/celestia-app/testutil/testfactory" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" coretypes "github.com/tendermint/tendermint/types" @@ -41,6 +45,67 @@ func Test_estimateSquareSize(t *testing.T) { } } +func Test_estimateSquareSize_MultiBlob(t *testing.T) { + enc := encoding.MakeConfig(ModuleEncodingRegisters...) + acc := "account" + kr := testfactory.GenerateKeyring(acc) + signer := blobtypes.NewKeyringSigner(kr, acc, "chainid") + type test struct { + name string + getBlobSizes func() [][]int + expectedSquareSize uint64 + expectedStartingShareIndex int + } + tests := []test{ + { + "single share multiblob transaction", + func() [][]int { return [][]int{{4}} }, + 2, 1, + }, + { + "10 multiblob single share transactions", + func() [][]int { + return blobfactory.Repeat([]int{100}, 10) + }, + 8, 7, + }, + { + "10 multiblob 2 share transactions", + func() [][]int { + return blobfactory.Repeat([]int{1000}, 10) + }, + 8, 7, + }, + { + "10 multiblob 4 share transactions", + func() [][]int { + return blobfactory.Repeat([]int{2000}, 10) + }, + 16, 7, + }, + { + "100 multiblob single share transaction", func() [][]int { + return [][]int{blobfactory.Repeat(int(100), 100)} + }, + 16, 5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + txs := blobfactory.ManyMultiBlobTxSameSigner( + t, + enc.TxConfig.TxEncoder(), + signer, + tt.getBlobSizes(), + ) + ptxs := parseTxs(enc.TxConfig, shares.TxsToBytes(txs)) + resSquareSize, resStart := estimateSquareSize(ptxs) + require.Equal(t, tt.expectedSquareSize, resSquareSize) + require.Equal(t, tt.expectedStartingShareIndex, resStart) + }) + } +} + func Test_estimateTxShares(t *testing.T) { type test struct { name string @@ -87,3 +152,44 @@ func Test_estimateTxShares(t *testing.T) { }) } } + +// The point of this test is to fail if anything to do with the serialization +// of index wrappers change, as changes could lead to tricky bugs. +func Test_expected_maxIndexWrapperOverhead(t *testing.T) { + assert.Equal(t, 2, maxIndexOverhead(4)) + assert.Equal(t, 5, maxIndexOverhead(128)) + assert.Equal(t, 6, maxIndexOverhead(512)) + assert.Equal(t, 12, maxIndexWrapperOverhead(4)) + assert.Equal(t, 16, maxIndexWrapperOverhead(128)) + assert.Equal(t, 16, maxIndexWrapperOverhead(512)) +} + +func Test_maxIndexWrapperOverhead(t *testing.T) { + type test struct { + squareSize int + blobs int + } + tests := []test{ + {4, 2}, + {32, 2}, + {128, 1}, + {128, 10}, + {128, 1000}, + {512, 4}, + } + for i, tt := range tests { + maxTxLen := tt.squareSize * tt.squareSize * appconsts.ContinuationCompactShareContentSize + blobLens := make([]uint32, tt.blobs) + for i := 0; i < tt.blobs; i++ { + blobLens[i] = uint32(tt.squareSize * tt.squareSize) + } + tx := make([]byte, maxTxLen) + wtx, err := coretypes.MarshalIndexWrapper(tx, blobLens...) + require.NoError(t, err) + + wrapperOverhead := maxIndexWrapperOverhead(uint64(tt.squareSize)) + indexOverhead := maxIndexOverhead(uint64(tt.squareSize)) * tt.blobs + + assert.LessOrEqual(t, len(wtx)-len(tx), wrapperOverhead+indexOverhead, i) + } +} diff --git a/app/parse_txs.go b/app/parse_txs.go index 61308db2a7..f201bf4343 100644 --- a/app/parse_txs.go +++ b/app/parse_txs.go @@ -16,8 +16,8 @@ type parsedTx struct { normalTx []byte // blobTx is the processed blob transaction. this field is only filled if // the transaction has blobs attached - blobTx core.BlobTx - shareIndex uint32 + blobTx core.BlobTx + shareIndexes []uint32 } // parseTxs decodes raw tendermint txs along with checking for and processing @@ -52,7 +52,7 @@ func processTxs(logger log.Logger, txs []parsedTx) [][]byte { // if this is a blob transaction, then we need to encode and wrap the // underlying MsgPFB containing transaction - wTx, err := coretypes.MarshalIndexWrapper(pTx.blobTx.Tx, pTx.shareIndex) + wTx, err := coretypes.MarshalIndexWrapper(pTx.blobTx.Tx, pTx.shareIndexes...) if err != nil { // note: Its not safe to bubble this error up and stop the block // creation process. diff --git a/app/prepare_proposal.go b/app/prepare_proposal.go index a8b190f17a..4692184a8f 100644 --- a/app/prepare_proposal.go +++ b/app/prepare_proposal.go @@ -9,7 +9,7 @@ import ( coretypes "github.com/tendermint/tendermint/types" ) -// PrepareProposal fullfills the celestia-core version of the ABCI interface by +// PrepareProposal fulfills the celestia-core version of the ABCI interface by // preparing the proposal block data. The square size is determined by first // estimating it via the size of the passed block data. Then, this method // generates the data root for the proposal block and passes it back to diff --git a/app/process_proposal.go b/app/process_proposal.go index 5a39707646..ed612d18bd 100644 --- a/app/process_proposal.go +++ b/app/process_proposal.go @@ -8,7 +8,6 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/inclusion" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/celestia-app/x/blob/types" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/rsmt2d" sdk "github.com/cosmos/cosmos-sdk/types" @@ -71,7 +70,6 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) abci.ResponsePr // iterate over all of the MsgPayForBlob transactions and ensure that their // commitments are subtree roots of the data root. - commitmentCounter := 0 for _, rawTx := range req.BlockData.Txs { tx := rawTx wrappedTx, isWrapped := coretypes.UnmarshalIndexWrapper(rawTx) @@ -118,7 +116,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) abci.ResponsePr } } - commitment, err := inclusion.GetMultiCommit(cacher, dah, wrappedTx.ShareIndexes, []uint32{pfb.BlobSize}) + commitment, err := inclusion.GetMultiCommit(cacher, dah, wrappedTx.ShareIndexes, pfb.BlobSizes) if err != nil { logInvalidPropBlockError(app.Logger(), req.Header, "commitment not found", err) return abci.ResponseProcessProposal{ @@ -132,18 +130,8 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) abci.ResponsePr Result: abci.ResponseProcessProposal_REJECT, } } - - commitmentCounter++ } - // compare the number of MPFBs and blobs, if they aren't - // identical, then we already know this block is invalid - if commitmentCounter != len(req.BlockData.Blobs) { - logInvalidPropBlock(app.Logger(), req.Header, "varying number of MsgPayForBlob and blobs in the same block") - return abci.ResponseProcessProposal{ - Result: abci.ResponseProcessProposal_REJECT, - } - } return abci.ResponseProcessProposal{ Result: abci.ResponseProcessProposal_ACCEPT, } @@ -151,7 +139,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) abci.ResponsePr func hasPFB(msgs []sdk.Msg) (*blobtypes.MsgPayForBlob, bool) { for _, msg := range msgs { - if pfb, ok := msg.(*types.MsgPayForBlob); ok { + if pfb, ok := msg.(*blobtypes.MsgPayForBlob); ok { return pfb, true } } diff --git a/app/test/block_size_test.go b/app/test/block_size_test.go index 5a5d3afd95..d402e0fb17 100644 --- a/app/test/block_size_test.go +++ b/app/test/block_size_test.go @@ -19,6 +19,7 @@ import ( "github.com/celestiaorg/celestia-app/testutil/blobfactory" "github.com/celestiaorg/celestia-app/testutil/network" "github.com/celestiaorg/celestia-app/x/blob" + "github.com/celestiaorg/celestia-app/x/blob/types" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" sdk "github.com/cosmos/cosmos-sdk/types" @@ -93,6 +94,23 @@ func (s *IntegrationTestSuite) TestMaxBlockSize() { s.kr, c.GRPCClient, 950000, + 1, + false, + s.cfg.ChainID, + s.accounts[:20], + ) + } + + // Tendermint's default tx size limit is 1 MiB, so we get close to that by + // generating transactions of size 600 KiB because 3 blobs per transaction * + // 200,000 bytes each = 600,000 total bytes = 600 KiB per transaction. + randMultiBlob1MbTxGen := func(c client.Context) []coretypes.Tx { + return blobfactory.RandBlobTxsWithAccounts( + s.cfg.TxConfig.TxEncoder(), + s.kr, + c.GRPCClient, + 200000, // 200 KiB + 3, false, s.cfg.ChainID, s.accounts[:20], @@ -108,10 +126,11 @@ func (s *IntegrationTestSuite) TestMaxBlockSize() { s.cfg.TxConfig.TxEncoder(), s.kr, c.GRPCClient, - 500000, + 50000, + 8, true, s.cfg.ChainID, - s.accounts[20:], + s.accounts[:80], ) } @@ -125,6 +144,10 @@ func (s *IntegrationTestSuite) TestMaxBlockSize() { "20 ~1Mb txs", equallySized1MbTxGen, }, + { + "20 ~1Mb multiblob txs", + randMultiBlob1MbTxGen, + }, { "80 random txs", randoTxGen, @@ -154,8 +177,11 @@ func (s *IntegrationTestSuite) TestMaxBlockSize() { for _, hash := range hashes { // TODO: reenable fetching and verifying proofs resp, err := queryTx(val.ClientCtx, hash, false) - require.NoError(err) - require.NotNil(resp) + assert.NoError(err) + assert.NotNil(resp) + if resp == nil { + continue + } assert.Equal(abci.CodeTypeOK, resp.TxResult.Code) heights[resp.Height]++ // ensure that some gas was used @@ -213,8 +239,8 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob() { }, { "large random typical", - mustNewBlob([]byte{2, 3, 4, 5, 6, 7, 8, 9}, tmrand.Bytes(700000)), - []blobtypes.TxBuilderOption{ + mustNewBlob([]byte{2, 3, 4, 5, 6, 7, 8, 9}, tmrand.Bytes(350000)), + []types.TxBuilderOption{ blobtypes.SetFeeAmount(sdk.NewCoins(sdk.NewCoin(app.BondDenom, sdk.NewInt(10)))), }, }, @@ -241,7 +267,7 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob() { require.NoError(s.network.WaitForNextBlock()) } signer := blobtypes.NewKeyringSigner(s.kr, s.accounts[0], val.ClientCtx.ChainID) - res, err := blob.SubmitPayForBlob(context.TODO(), signer, val.ClientCtx.GRPCClient, tc.blob, 1000000000, tc.opts...) + res, err := blob.SubmitPayForBlob(context.TODO(), signer, val.ClientCtx.GRPCClient, []*blobtypes.Blob{tc.blob, tc.blob}, 1000000000, tc.opts...) require.NoError(err) require.NotNil(res) assert.Equal(abci.CodeTypeOK, res.Code) @@ -257,7 +283,8 @@ func (s *IntegrationTestSuite) TestUnwrappedPFBRejection() { s.cfg.TxConfig.TxEncoder(), s.kr, val.ClientCtx.GRPCClient, - 100000, + int(100000), + 1, false, s.cfg.ChainID, s.accounts[:1], @@ -268,7 +295,7 @@ func (s *IntegrationTestSuite) TestUnwrappedPFBRejection() { res, err := val.ClientCtx.BroadcastTxSync(btx.Tx) require.NoError(t, err) - require.Equal(t, blobtypes.ErrBloblessPFB.ABCICode(), res.Code) + require.Equal(t, blobtypes.ErrNoBlobs.ABCICode(), res.Code) } func queryTx(clientCtx client.Context, hashHexStr string, prove bool) (*rpctypes.ResultTx, error) { diff --git a/app/test/check_tx_test.go b/app/test/check_tx_test.go index 7d3437da8f..dfcd7d581c 100644 --- a/app/test/check_tx_test.go +++ b/app/test/check_tx_test.go @@ -5,9 +5,9 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/testutil" "github.com/celestiaorg/celestia-app/testutil/blobfactory" + "github.com/celestiaorg/celestia-app/testutil/namespace" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -20,7 +20,7 @@ import ( func TestCheckTx(t *testing.T) { encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) - accs := []string{"a", "b", "c", "d"} + accs := []string{"a", "b", "c", "d", "e", "f"} testApp, kr := testutil.SetupTestAppWithGenesisValSet(accs...) @@ -78,7 +78,7 @@ func TestCheckTx(t *testing.T) { )[0] dtx, _ := coretypes.UnmarshalBlobTx(btx) - dtx.Blobs[0].NamespaceId = appconsts.TxNamespaceID + dtx.Blobs[0].NamespaceId = namespace.RandomBlobNamespace() bbtx, err := coretypes.MarshalBlobTx(dtx.Tx, dtx.Blobs[0]) require.NoError(t, err) return bbtx @@ -100,7 +100,16 @@ func TestCheckTx(t *testing.T) { dtx, _ := coretypes.UnmarshalBlobTx(btx) return dtx.Tx }, - expectedABCICode: blobtypes.ErrBloblessPFB.ABCICode(), + expectedABCICode: blobtypes.ErrNoBlobs.ABCICode(), + }, + { + name: "normal blobTx w/ multiple blobs, CheckTxType_New", + checkType: abci.CheckTxType_New, + getTx: func() []byte { + tx := blobfactory.RandBlobTxsWithAccounts(encCfg.TxConfig.TxEncoder(), kr, nil, 10000, 10, true, testutil.ChainID, accs[3:4])[0] + return tx + }, + expectedABCICode: abci.CodeTypeOK, }, } diff --git a/app/test/fuzz_abci_test.go b/app/test/fuzz_abci_test.go index abebb92bde..5e56767938 100644 --- a/app/test/fuzz_abci_test.go +++ b/app/test/fuzz_abci_test.go @@ -19,31 +19,36 @@ import ( // blocks produced by PrepareProposal should be accepted by ProcessProposal. It // doesn't use the standard go tools for fuzzing as those tools only support // fuzzing limited types, instead we create blocks our selves using random -// transction. +// transactions. func TestPrepareProposalConsistency(t *testing.T) { if testing.Short() { t.Skip("skipping TestPrepareProposalConsistency in short mode.") } encConf := encoding.MakeConfig(app.ModuleEncodingRegisters...) testApp, _ := testutil.SetupTestAppWithGenesisValSet() - timer := time.After(time.Minute * 1) type test struct { - count, size int + name string + count, blobCount, size int + } + tests := []test{ + {"many small single share single blob transactions", 10000, 1, 400}, + {"one hundred normal sized single blob transactions", 100, 1, 400000}, + {"many single share multi-blob transactions", 1000, 100, 400}, + {"one hundred normal sized multi-blob transactions", 100, 4, 400000}, } - tests := []test{{10000, 400}, {100, 400000}} - for _, tt := range tests { - for { - select { - case <-timer: - return - default: - t.Run("randomized inputs to Prepare and Process Proposal", func(t *testing.T) { - ProcessRandomProposal(t, tt.count, tt.size, encConf, testApp) - }) + t.Run(tt.name, func(t *testing.T) { + timer := time.After(time.Second * 30) + for { + select { + case <-timer: + return + default: + ProcessRandomProposal(t, tt.count, tt.size, tt.blobCount, encConf, testApp) + } } - } + }) } } @@ -51,10 +56,11 @@ func ProcessRandomProposal( t *testing.T, count, maxSize int, + maxBlobCount int, cfg encoding.Config, capp *app.App, ) { - txs := blobfactory.RandBlobTxsRandomlySized(cfg.TxConfig.TxEncoder(), count, maxSize) + txs := blobfactory.RandBlobTxsRandomlySized(cfg.TxConfig.TxEncoder(), count, maxSize, maxBlobCount) sendTxs := blobfactory.GenerateManyRawSendTxs(cfg.TxConfig, count) txs = append(txs, sendTxs...) resp := capp.PrepareProposal(abci.RequestPrepareProposal{ diff --git a/app/testutil_test.go b/app/testutil_test.go index 9b44160d9e..057d6efdae 100644 --- a/app/testutil_test.go +++ b/app/testutil_test.go @@ -1,10 +1,13 @@ package app import ( + "testing" + "github.com/celestiaorg/celestia-app/app/encoding" coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/testutil/blobfactory" + "github.com/celestiaorg/celestia-app/testutil/testfactory" ) func generateMixedParsedTxs(normalTxCount, pfbCount, pfbSize int) []parsedTx { @@ -19,8 +22,19 @@ func generateMixedParsedTxs(normalTxCount, pfbCount, pfbSize int) []parsedTx { return parseTxs(encCfg.TxConfig, coretypes.Txs(txs).ToSliceOfBytes()) } -func generateParsedTxsWithNIDs(nids [][]byte, sizes []int) []parsedTx { +// generateParsedTxsWithNIDs will generate len(nids) parsed BlobTxs with +// len(blobSizes[i]) number of blobs per BlobTx. +func generateParsedTxsWithNIDs(t *testing.T, nids [][]byte, blobSizes [][]int) []parsedTx { encCfg := encoding.MakeConfig(ModuleEncodingRegisters...) - txs := blobfactory.RandBlobTxsWithNamespaces(encCfg.TxConfig.TxEncoder(), nids, sizes) - return parseTxs(encCfg.TxConfig, coretypes.Txs(txs).ToSliceOfBytes()) + const acc = "signer" + kr := testfactory.GenerateKeyring(acc) + txs := blobfactory.ManyMultiBlobTx( + t, + encCfg.TxConfig.TxEncoder(), + kr, + "chainid", + blobfactory.Repeat(acc, len(blobSizes)), + blobfactory.NestedBlobs(t, nids, blobSizes), + ) + return parseTxs(encCfg.TxConfig, txs) } diff --git a/app/write_square.go b/app/write_square.go index 42a1cd9784..53fcd66413 100644 --- a/app/write_square.go +++ b/app/write_square.go @@ -2,7 +2,6 @@ package app import ( "bytes" - "fmt" "sort" "github.com/celestiaorg/celestia-app/pkg/shares" @@ -10,9 +9,13 @@ import ( ) type trackedBlob struct { - blob tmproto.Blob + blob *tmproto.Blob + // parsedIndex keeps track of which parsed transaction a blob relates to parsedIndex int - sharesUsed int + // blobIndex keeps track of which blob in a parsed transaction + // this blob relates to + blobIndex int + sharesUsed int } // finalizeLayout returns the transactions and blobs in their completed layout. @@ -29,12 +32,15 @@ func finalizeLayout(squareSize uint64, nonreserveStart int, ptxs []parsedTx) ([] if len(pTx.normalTx) != 0 { continue } - dataUsed := len(pTx.blobTx.Blobs[0].Data) - trackedBlobs = append(trackedBlobs, trackedBlob{ - blob: *pTx.blobTx.Blobs[0], - parsedIndex: i, - sharesUsed: shares.SparseSharesNeeded(uint32(dataUsed)), - }) + ptxs[i].shareIndexes = make([]uint32, len(pTx.blobTx.Blobs)) + for j, blob := range pTx.blobTx.Blobs { + trackedBlobs = append(trackedBlobs, trackedBlob{ + blob: blob, + parsedIndex: i, + blobIndex: j, + sharesUsed: shares.SparseSharesNeeded(uint32(len(blob.Data))), + }) + } } // blobs must be sorted by namespace in order for nmt to be able to create a @@ -46,21 +52,24 @@ func finalizeLayout(squareSize uint64, nonreserveStart int, ptxs []parsedTx) ([] cursor := nonreserveStart iSS := int(squareSize) maxSharesSize := iSS * iSS - blobs := make([]tmproto.Blob, 0) - removeList := []int{} + removeIndexes := make(map[int]bool) for _, tBlob := range trackedBlobs { + // skip this blob, as it will already be removed + if removeIndexes[tBlob.parsedIndex] { + continue + } cursor, _ = shares.NextMultipleOfBlobMinSquareSize(cursor, tBlob.sharesUsed, iSS) // remove the parsed transaction if it cannot fit into the square if cursor+tBlob.sharesUsed > maxSharesSize { - removeList = append(removeList, tBlob.parsedIndex) + removeIndexes[tBlob.parsedIndex] = true continue } - ptxs[tBlob.parsedIndex].shareIndex = uint32(cursor) - blobs = append(blobs, tBlob.blob) + // set the share index in the same order as the blob that its for + ptxs[tBlob.parsedIndex].shareIndexes[tBlob.blobIndex] = uint32(cursor) cursor += tBlob.sharesUsed } - ptxs = removeMany(ptxs, removeList...) + ptxs = removeMany(ptxs, removeIndexes) blobTxCount := 0 for _, ptx := range ptxs { @@ -69,27 +78,32 @@ func finalizeLayout(squareSize uint64, nonreserveStart int, ptxs []parsedTx) ([] } } - if blobTxCount != len(blobs) { - panic(fmt.Sprintf("invalid number of blob txs: must be equal to number of blobs: txs %d blobs %d", blobTxCount, len(blobs))) + derefBlobs := make([]tmproto.Blob, 0) + for _, ptx := range ptxs { + if len(ptx.normalTx) != 0 { + continue + } + for _, blob := range ptx.blobTx.Blobs { + derefBlobs = append(derefBlobs, *blob) + } } - return ptxs, blobs -} + // todo: don't sort twice + sort.SliceStable(derefBlobs, func(i, j int) bool { + return bytes.Compare(derefBlobs[i].NamespaceId, derefBlobs[j].NamespaceId) < 0 + }) -func removeMany[T any](s []T, indexes ...int) []T { - // Create a map to track which indexes to remove - remove := make(map[int]bool) - for _, i := range indexes { - remove[i] = true - } + return ptxs, derefBlobs +} +func removeMany[T any](s []T, removeIndexes map[int]bool) []T { // Create a new slice to store the remaining elements - result := make([]T, 0, len(s)-len(indexes)) + result := make([]T, 0, len(s)-len(removeIndexes)) // Iterate over the original slice and append elements // to the result slice unless they are marked for removal for i, x := range s { - if !remove[i] { + if !removeIndexes[i] { result = append(result, x) } } diff --git a/app/write_square_test.go b/app/write_square_test.go index e551dcf22f..0a88cb8919 100644 --- a/app/write_square_test.go +++ b/app/write_square_test.go @@ -6,10 +6,11 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/celestia-app/testutil/blobfactory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmlog "github.com/tendermint/tendermint/libs/log" - core "github.com/tendermint/tendermint/proto/tendermint/types" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" ) @@ -22,119 +23,158 @@ func Test_finalizeLayout(t *testing.T) { squareSize uint64 nonreserveStart int ptxs []parsedTx - expectedIndexes []uint32 + expectedIndexes [][]uint32 } tests := []test{ { squareSize: 4, nonreserveStart: 10, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1}, - []int{1}, + [][]int{{1}}, ), - expectedIndexes: []uint32{10}, + expectedIndexes: [][]uint32{{10}}, }, { squareSize: 4, nonreserveStart: 10, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns1}, - []int{100, 100}, + blobfactory.Repeat([]int{100}, 2), ), - expectedIndexes: []uint32{10, 11}, + expectedIndexes: [][]uint32{{10}, {11}}, }, { squareSize: 4, nonreserveStart: 10, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns1, ns1, ns1, ns1, ns1, ns1, ns1, ns1, ns1}, - []int{100, 100, 100, 100, 100, 100, 100, 100, 100, 100}, + blobfactory.Repeat([]int{100}, 10), ), - expectedIndexes: []uint32{10, 11, 12, 13, 14, 15}, + expectedIndexes: [][]uint32{{10}, {11}, {12}, {13}, {14}, {15}}, }, { squareSize: 4, nonreserveStart: 7, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns1, ns1, ns1, ns1, ns1, ns1, ns1, ns1}, - []int{100, 100, 100, 100, 100, 100, 100, 100, 100}, + blobfactory.Repeat([]int{100}, 9), ), - expectedIndexes: []uint32{7, 8, 9, 10, 11, 12, 13, 14, 15}, + expectedIndexes: [][]uint32{{7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, {15}}, }, { squareSize: 4, nonreserveStart: 3, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns1, ns1}, - []int{10000, 10000, 1000000}, + [][]int{{10000}, {10000}, {1000000}}, ), - expectedIndexes: []uint32{}, + expectedIndexes: [][]uint32{}, }, { squareSize: 64, nonreserveStart: 32, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns1, ns1}, - []int{1000, 10000, 100000}, // blob share lengths of 2, 20, 199 respectively + [][]int{{1000}, {10000}, {100000}}, ), - expectedIndexes: []uint32{ + expectedIndexes: [][]uint32{ // BlobMinSquareSize(2) = 2 so the first blob has to start at the // next multiple of 2 >= 32 which is 32. This blob occupies // shares 32 to 33. - 32, + {32}, // BlobMinSquareSize(20) = 8 so the second blob has to start at // the next multiple of 8 >= 34 which is 40. This blob occupies // shares 40 to 59. - 40, + {40}, // BlobMinSquareSize(199) = 16 so the third blob has to start at // the next multiple of 16 >= 60 which is 64. This blob occupies // shares 64 to 262. - 64, + {64}, }, }, { squareSize: 32, nonreserveStart: 32, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns2, ns1, ns1}, - []int{100, 100, 100}, + [][]int{{100}, {100}, {100}}, ), - expectedIndexes: []uint32{34, 32, 33}, + expectedIndexes: [][]uint32{{34}, {32}, {33}}, }, { squareSize: 32, nonreserveStart: 32, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns2, ns1}, - []int{100, 1000, 1000}, + [][]int{{100}, {1000}, {1000}}, ), - expectedIndexes: []uint32{32, 36, 34}, + expectedIndexes: [][]uint32{{32}, {36}, {34}}, }, { squareSize: 32, nonreserveStart: 32, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns2, ns1}, - []int{100, 1000, 1000}, + [][]int{{100}, {1000}, {1000}}, ), - expectedIndexes: []uint32{32, 36, 34}, + expectedIndexes: [][]uint32{{32}, {36}, {34}}, }, { squareSize: 4, nonreserveStart: 2, ptxs: generateParsedTxsWithNIDs( + t, [][]byte{ns1, ns3, ns2}, - []int{100, 1000, 420}, + [][]int{{100}, {1000}, {420}}, ), - expectedIndexes: []uint32{2, 4, 3}, + expectedIndexes: [][]uint32{{2}, {4}, {3}}, + }, + { + squareSize: 4, + nonreserveStart: 4, + ptxs: generateParsedTxsWithNIDs( + t, + [][]byte{ns1, ns3, ns3, ns2}, + [][]int{{100}, {1000, 1000}, {420}}, + ), + expectedIndexes: [][]uint32{{4}, {6, 8}, {5}}, + }, + { + squareSize: 4, + nonreserveStart: 4, + ptxs: generateParsedTxsWithNIDs( + t, + [][]byte{ns1, ns3, ns3, ns1, ns2, ns2}, + [][]int{{100}, {1400, 1000, 200, 200}, {420}}, + ), + expectedIndexes: [][]uint32{{4}, {8, 12, 5, 6}, {7}}, + }, + { + squareSize: 4, + nonreserveStart: 4, + ptxs: generateParsedTxsWithNIDs( + t, + [][]byte{ns1, ns3, ns3, ns1, ns2, ns2}, + [][]int{{100}, {1000, 1400, 200, 200}, {420}}, + ), + expectedIndexes: [][]uint32{{4}, {8, 10, 5, 6}, {7}}, }, } for i, tt := range tests { res, blobs := finalizeLayout(tt.squareSize, tt.nonreserveStart, tt.ptxs) require.Equal(t, len(tt.expectedIndexes), len(res), i) - require.Equal(t, len(tt.expectedIndexes), len(blobs), i) - for i, ptx := range res { - assert.Equal(t, tt.expectedIndexes[i], ptx.shareIndex, i) + for j, ptx := range res { + assert.Equal(t, tt.expectedIndexes[j], ptx.shareIndexes, i) } processedTxs := processTxs(tmlog.NewNopLogger(), res) @@ -143,7 +183,7 @@ func Test_finalizeLayout(t *testing.T) { return bytes.Compare(blobs[i].NamespaceId, blobs[j].NamespaceId) < 0 }) - blockData := core.Data{ + blockData := tmproto.Data{ Txs: processedTxs, Blobs: blobs, SquareSize: tt.squareSize, diff --git a/pkg/shares/share_splitting.go b/pkg/shares/share_splitting.go index c0d8fe2356..73823c8e0d 100644 --- a/pkg/shares/share_splitting.go +++ b/pkg/shares/share_splitting.go @@ -11,7 +11,7 @@ import ( var ( ErrIncorrectNumberOfIndexes = errors.New( - "number of malleated transactions is not identical to the number of wrapped transactions", + "number of indexes is not identical to the number of blobs", ) ErrUnexpectedFirstBlobShareIndex = errors.New( "the first blob started at an unexpected index", @@ -78,7 +78,7 @@ func Split(data coretypes.Data, useShareIndexes bool) ([]Share, error) { func ExtractShareIndexes(txs coretypes.Txs) []uint32 { var shareIndexes []uint32 for _, rawTx := range txs { - if malleatedTx, isMalleated := coretypes.UnmarshalIndexWrapper(rawTx); isMalleated { + if indexWrappedTxs, isIndexWrapped := coretypes.UnmarshalIndexWrapper(rawTx); isIndexWrapped { // Since share index == 0 is invalid, it indicates that we are // attempting to extract share indexes from txs that do not have any // due to them being old. here we return nil to indicate that we are @@ -86,10 +86,10 @@ func ExtractShareIndexes(txs coretypes.Txs) []uint32 { // it. It checks for 0 because if there is a message in the block, // then there must also be a tx, which will take up at least one // share. - if len(malleatedTx.ShareIndexes) == 0 { + if len(indexWrappedTxs.ShareIndexes) == 0 { return nil } - shareIndexes = append(shareIndexes, malleatedTx.ShareIndexes...) + shareIndexes = append(shareIndexes, indexWrappedTxs.ShareIndexes...) } } diff --git a/proto/blob/event.proto b/proto/blob/event.proto index 0913b71da3..9911267912 100644 --- a/proto/blob/event.proto +++ b/proto/blob/event.proto @@ -10,5 +10,5 @@ option go_package = "github.com/celestiaorg/celestia-app/x/blob/types"; message EventPayForBlob { string signer = 1; uint32 blob_size = 2; - bytes namespace_id = 3; + repeated bytes namespace_ids = 3; } diff --git a/proto/blob/tx.proto b/proto/blob/tx.proto index b7cc33d839..221ce15dd5 100644 --- a/proto/blob/tx.proto +++ b/proto/blob/tx.proto @@ -26,16 +26,16 @@ message ShareCommitAndSignature { // MsgPayForBlob pays for the inclusion of a blob in the block. message MsgPayForBlob { string signer = 1; - bytes namespace_id = 2; - uint32 blob_size = 3; + repeated bytes namespace_ids = 2; + repeated uint32 blob_sizes = 3; // share_commitment is the share_commitment from // ShareCommitAndSignature that will be included in a block bytes share_commitment = 4; - // share_version is the version of the share format that the blob associated - // with this message should use when included in a block. The share_version - // specified must match the share_version used to generate the + // share_versions are the versions of the share format that the blobs + // associated with this message should use when included in a block. The + // share_versions specified must match the share_versions used to generate the // share_commitment in this message. - uint32 share_version = 8; + repeated uint32 share_versions = 8; } // MsgPayForBlobResponse describes the response returned after the submission diff --git a/testutil/blobfactory/payforblob_factory.go b/testutil/blobfactory/payforblob_factory.go index efa600df9a..4aa41523b1 100644 --- a/testutil/blobfactory/payforblob_factory.go +++ b/testutil/blobfactory/payforblob_factory.go @@ -4,9 +4,9 @@ import ( "context" "testing" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/testutil/namespace" "github.com/celestiaorg/celestia-app/testutil/testfactory" + "github.com/celestiaorg/celestia-app/x/blob/types" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" @@ -20,31 +20,46 @@ import ( var defaultSigner = testfactory.RandomAddress().String() -func RandMsgPayForBlobWithSigner(singer string, size int) (*blobtypes.MsgPayForBlob, []byte) { - blob := tmrand.Bytes(size) +func RandMsgPayForBlobWithSigner(singer string, size, blobCount int) (*blobtypes.MsgPayForBlob, []*tmproto.Blob) { + blobs := make([]*tmproto.Blob, blobCount) + for i := 0; i < blobCount; i++ { + blob, err := types.NewBlob(namespace.RandomBlobNamespace(), tmrand.Bytes(size)) + if err != nil { + panic(err) + } + blobs[i] = blob + } + msg, err := blobtypes.NewMsgPayForBlob( singer, - &tmproto.Blob{ - NamespaceId: namespace.RandomBlobNamespace(), - Data: blob, - ShareVersion: uint32(appconsts.ShareVersionZero), - }, + blobs..., ) if err != nil { panic(err) } - return msg, blob + return msg, blobs +} + +func RandBlobsWithNamespace(namespaces [][]byte, sizes []int) []*tmproto.Blob { + blobs := make([]*tmproto.Blob, len(namespaces)) + for i, ns := range namespaces { + blob, err := types.NewBlob(ns, tmrand.Bytes(sizes[i])) + if err != nil { + panic(err) + } + blobs[i] = blob + } + return blobs } -func RandMsgPayForBlobWithNamespaceAndSigner(signer string, nid []byte, size int) (*blobtypes.MsgPayForBlob, []byte) { - blob := tmrand.Bytes(size) +func RandMsgPayForBlobWithNamespaceAndSigner(signer string, nid []byte, size int) (*blobtypes.MsgPayForBlob, *tmproto.Blob) { + blob, err := types.NewBlob(nid, tmrand.Bytes(size)) + if err != nil { + panic(err) + } msg, err := blobtypes.NewMsgPayForBlob( signer, - &tmproto.Blob{ - NamespaceId: nid, - Data: blob, - ShareVersion: uint32(appconsts.ShareVersionZero), - }, + blob, ) if err != nil { panic(err) @@ -52,15 +67,14 @@ func RandMsgPayForBlobWithNamespaceAndSigner(signer string, nid []byte, size int return msg, blob } -func RandMsgPayForBlob(size int) (*blobtypes.MsgPayForBlob, []byte) { - blob := tmrand.Bytes(size) +func RandMsgPayForBlob(size int) (*blobtypes.MsgPayForBlob, *tmproto.Blob) { + blob, err := types.NewBlob(namespace.RandomBlobNamespace(), tmrand.Bytes(size)) + if err != nil { + panic(err) + } msg, err := blobtypes.NewMsgPayForBlob( defaultSigner, - &tmproto.Blob{ - NamespaceId: namespace.RandomBlobNamespace(), - Data: blob, - ShareVersion: uint32(appconsts.ShareVersionZero), - }, + blob, ) if err != nil { panic(err) @@ -68,7 +82,7 @@ func RandMsgPayForBlob(size int) (*blobtypes.MsgPayForBlob, []byte) { return msg, blob } -func RandBlobTxsRandomlySized(enc sdk.TxEncoder, count, maxSize int) []coretypes.Tx { +func RandBlobTxsRandomlySized(enc sdk.TxEncoder, count, maxSize, maxBlobs int) []coretypes.Tx { const acc = "signer" kr := testfactory.GenerateKeyring(acc) signer := blobtypes.NewKeyringSigner(kr, acc, "chainid") @@ -84,7 +98,7 @@ func RandBlobTxsRandomlySized(enc sdk.TxEncoder, count, maxSize int) []coretypes opts := []blobtypes.TxBuilderOption{ blobtypes.SetFeeAmount(sdk.NewCoins(coin)), - blobtypes.SetGasLimit(10000000), + blobtypes.SetGasLimit(100000000), } txs := make([]coretypes.Tx, count) @@ -94,7 +108,11 @@ func RandBlobTxsRandomlySized(enc sdk.TxEncoder, count, maxSize int) []coretypes if size == 0 { size = 1 } - msg, blob := RandMsgPayForBlobWithSigner(addr.String(), size) + blobCount := tmrand.Intn(maxBlobs) + if blobCount == 0 { + blobCount = 1 + } + msg, blobs := RandMsgPayForBlobWithSigner(addr.String(), size, blobCount) builder := signer.NewTxBuilder(opts...) stx, err := signer.BuildSignedTx(builder, msg) if err != nil { @@ -104,11 +122,7 @@ func RandBlobTxsRandomlySized(enc sdk.TxEncoder, count, maxSize int) []coretypes if err != nil { panic(err) } - wblob, err := blobtypes.NewBlob(msg.NamespaceId, blob) - if err != nil { - panic(err) - } - cTx, err := coretypes.MarshalBlobTx(rawTx, wblob) + cTx, err := coretypes.MarshalBlobTx(rawTx, blobs...) if err != nil { panic(err) } @@ -118,11 +132,16 @@ func RandBlobTxsRandomlySized(enc sdk.TxEncoder, count, maxSize int) []coretypes return txs } +// RandBlobTxsWithAccounts will create random blob transactions using the +// provided configuration. If no grpc connection is provided, then it will not +// update the account info. One blob transaction is generated per account +// provided. func RandBlobTxsWithAccounts( enc sdk.TxEncoder, kr keyring.Keyring, conn *grpc.ClientConn, size int, + blobCount int, randSize bool, chainid string, accounts []string, @@ -140,9 +159,11 @@ func RandBlobTxsWithAccounts( txs := make([]coretypes.Tx, len(accounts)) for i := 0; i < len(accounts); i++ { signer := blobtypes.NewKeyringSigner(kr, accounts[i], chainid) - err := signer.QueryAccountNumber(context.Background(), conn) - if err != nil { - panic(err) + if conn != nil { + err := signer.QueryAccountNumber(context.Background(), conn) + if err != nil { + panic(err) + } } addr, err := signer.GetSignerInfo().GetAddress() @@ -160,7 +181,14 @@ func RandBlobTxsWithAccounts( randomizedSize = 1 } } - msg, blob := RandMsgPayForBlobWithSigner(addr.String(), randomizedSize) + randomizedBlobCount := blobCount + if randSize { + randomizedBlobCount = rand.Intn(blobCount) + if randomizedBlobCount == 0 { + randomizedBlobCount = 1 + } + } + msg, blobs := RandMsgPayForBlobWithSigner(addr.String(), randomizedSize, randomizedBlobCount) builder := signer.NewTxBuilder(opts...) stx, err := signer.BuildSignedTx(builder, msg) if err != nil { @@ -170,11 +198,7 @@ func RandBlobTxsWithAccounts( if err != nil { panic(err) } - wblob, err := blobtypes.NewBlob(msg.NamespaceId, blob) - if err != nil { - panic(err) - } - cTx, err := coretypes.MarshalBlobTx(rawTx, wblob) + cTx, err := coretypes.MarshalBlobTx(rawTx, blobs...) if err != nil { panic(err) } @@ -205,7 +229,7 @@ func RandBlobTxs(enc sdk.TxEncoder, count, size int) []coretypes.Tx { txs := make([]coretypes.Tx, count) for i := 0; i < count; i++ { - msg, blob := RandMsgPayForBlobWithSigner(addr.String(), size) + msg, blobs := RandMsgPayForBlobWithSigner(addr.String(), size, 1) builder := signer.NewTxBuilder(opts...) stx, err := signer.BuildSignedTx(builder, msg) if err != nil { @@ -215,11 +239,7 @@ func RandBlobTxs(enc sdk.TxEncoder, count, size int) []coretypes.Tx { if err != nil { panic(err) } - wblob, err := blobtypes.NewBlob(msg.NamespaceId, blob) - if err != nil { - panic(err) - } - cTx, err := coretypes.MarshalBlobTx(rawTx, wblob) + cTx, err := coretypes.MarshalBlobTx(rawTx, blobs...) if err != nil { panic(err) } @@ -236,6 +256,114 @@ func RandBlobTxsWithNamespaces(enc sdk.TxEncoder, nIds [][]byte, sizes []int) [] return RandBlobTxsWithNamespacesAndSigner(enc, signer, nIds, sizes) } +// ManyMultiBlobTxSameSigner generates and returns many blob transactions with the +// possibility to add more than one blob. +func ManyMultiBlobTxSameSigner( + t *testing.T, + enc sdk.TxEncoder, + signer *blobtypes.KeyringSigner, + blobSizes [][]int, +) []coretypes.Tx { + txs := make([]coretypes.Tx, len(blobSizes)) + for i := 0; i < len(blobSizes); i++ { + txs[i] = MultiBlobTx(t, enc, signer, ManyRandBlobs(t, blobSizes[i]...)...) + } + return txs +} + +func ManyRandBlobsIdenticallySized(t *testing.T, count, size int) []*tmproto.Blob { + sizes := make([]int, count) + for i := 0; i < count; i++ { + sizes[i] = size + } + return ManyRandBlobs(t, sizes...) +} + +func ManyRandBlobs(t *testing.T, sizes ...int) []*tmproto.Blob { + return ManyBlobs(t, namespace.RandomBlobNamespaces(len(sizes)), sizes) +} + +func Repeat[T any](s T, count int) []T { + ss := make([]T, count) + for i := 0; i < count; i++ { + ss[i] = s + } + return ss +} + +func ManyBlobs(t *testing.T, namespaces [][]byte, sizes []int) []*tmproto.Blob { + blobs := make([]*tmproto.Blob, len(namespaces)) + for i, ns := range namespaces { + blob, err := blobtypes.NewBlob(ns, tmrand.Bytes(sizes[i])) + require.NoError(t, err) + blobs[i] = blob + } + return blobs +} + +func NestedBlobs(t *testing.T, nids [][]byte, sizes [][]int) [][]*tmproto.Blob { + blobs := make([][]*tmproto.Blob, len(sizes)) + counter := 0 + for i, set := range sizes { + for _, size := range set { + blob, err := blobtypes.NewBlob(nids[counter], tmrand.Bytes(size)) + require.NoError(t, err) + blobs[i] = append(blobs[i], blob) + counter++ + } + } + return blobs +} + +func ManyMultiBlobTx( + t *testing.T, + enc sdk.TxEncoder, + kr keyring.Keyring, + chainid string, + accounts []string, + blobs [][]*tmproto.Blob, +) [][]byte { + txs := make([][]byte, len(accounts)) + for i, acc := range accounts { + signer := blobtypes.NewKeyringSigner(kr, acc, chainid) + txs[i] = MultiBlobTx(t, enc, signer, blobs[i]...) + } + return txs +} + +func MultiBlobTx( + t *testing.T, + enc sdk.TxEncoder, + signer *blobtypes.KeyringSigner, + blobs ...*tmproto.Blob, +) coretypes.Tx { + addr, err := signer.GetSignerInfo().GetAddress() + require.NoError(t, err) + + coin := sdk.Coin{ + Denom: bondDenom, + Amount: sdk.NewInt(10), + } + opts := []blobtypes.TxBuilderOption{ + blobtypes.SetFeeAmount(sdk.NewCoins(coin)), + blobtypes.SetGasLimit(10000000), + } + msg, err := blobtypes.NewMsgPayForBlob(addr.String(), blobs...) + require.NoError(t, err) + + builder := signer.NewTxBuilder(opts...) + stx, err := signer.BuildSignedTx(builder, msg) + require.NoError(t, err) + + rawTx, err := enc(stx) + require.NoError(t, err) + + cTx, err := coretypes.MarshalBlobTx(rawTx, blobs...) + require.NoError(t, err) + + return cTx +} + func RandBlobTxsWithNamespacesAndSigner( enc sdk.TxEncoder, signer *blobtypes.KeyringSigner, @@ -269,11 +397,7 @@ func RandBlobTxsWithNamespacesAndSigner( if err != nil { panic(err) } - wblob, err := blobtypes.NewBlob(msg.NamespaceId, blob) - if err != nil { - panic(err) - } - cTx, err := coretypes.MarshalBlobTx(rawTx, wblob) + cTx, err := coretypes.MarshalBlobTx(rawTx, blob) if err != nil { panic(err) } @@ -288,7 +412,7 @@ func ComplexBlobTxWithOtherMsgs(t *testing.T, kr keyring.Keyring, enc sdk.TxEnco signerAddr, err := signer.GetSignerInfo().GetAddress() require.NoError(t, err) - pfb, rawBlob := RandMsgPayForBlobWithSigner(signerAddr.String(), 100) + pfb, blobs := RandMsgPayForBlobWithSigner(signerAddr.String(), 100, 1) opts := []blobtypes.TxBuilderOption{ blobtypes.SetFeeAmount(sdk.NewCoins(sdk.NewCoin(bondDenom, sdk.NewInt(10)))), @@ -302,10 +426,7 @@ func ComplexBlobTxWithOtherMsgs(t *testing.T, kr keyring.Keyring, enc sdk.TxEnco rawTx, err := enc(sdkTx) require.NoError(t, err) - blob, err := blobtypes.NewBlob(pfb.NamespaceId, rawBlob) - require.NoError(t, err) - - btx, err := coretypes.MarshalBlobTx(rawTx, blob) + btx, err := coretypes.MarshalBlobTx(rawTx, blobs...) require.NoError(t, err) return btx } diff --git a/testutil/namespace/random_ns_generator.go b/testutil/namespace/random_ns_generator.go index 4348f14fe8..b2503dd428 100644 --- a/testutil/namespace/random_ns_generator.go +++ b/testutil/namespace/random_ns_generator.go @@ -20,3 +20,11 @@ func RandomBlobNamespace() nmtnamespace.ID { return ns } } + +func RandomBlobNamespaces(count int) [][]byte { + namespaces := make([][]byte, count) + for i := 0; i < count; i++ { + namespaces[i] = RandomBlobNamespace() + } + return namespaces +} diff --git a/testutil/testnode/node_interaction_api.go b/testutil/testnode/node_interaction_api.go index ae03a5b294..1f386615c3 100644 --- a/testutil/testnode/node_interaction_api.go +++ b/testutil/testnode/node_interaction_api.go @@ -14,7 +14,6 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" abci "github.com/tendermint/tendermint/abci/types" tmrand "github.com/tendermint/tendermint/libs/rand" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" ) @@ -112,13 +111,14 @@ func (c *Context) PostData(account, broadcastMode string, ns, blobData []byte) ( signer.SetAccountNumber(acc) signer.SetSequence(seq) + blob, err := types.NewBlob(ns, blobData) + if err != nil { + return nil, err + } + msg, err := types.NewMsgPayForBlob( addr.String(), - &tmproto.Blob{ - NamespaceId: ns, - Data: blobData, - ShareVersion: uint32(appconsts.ShareVersionZero), - }, + blob, ) if err != nil { return nil, err @@ -129,15 +129,13 @@ func (c *Context) PostData(account, broadcastMode string, ns, blobData []byte) ( if err != nil { return nil, err } - wblob, err := types.NewBlob(msg.NamespaceId, blobData) - if err != nil { - return nil, err - } + rawTx, err := signer.EncodeTx(stx) if err != nil { return nil, err } - blobTx, err := coretypes.MarshalBlobTx(rawTx, wblob) + + blobTx, err := coretypes.MarshalBlobTx(rawTx, blob) if err != nil { return nil, err } diff --git a/x/blob/client/cli/wirepayfordata.go b/x/blob/client/cli/wirepayfordata.go index 33832d69a2..2165a2f057 100644 --- a/x/blob/client/cli/wirepayfordata.go +++ b/x/blob/client/cli/wirepayfordata.go @@ -41,6 +41,7 @@ func CmdWirePayForBlob() *cobra.Command { return fmt.Errorf("failure to decode hex blob: %w", err) } + // TODO: allow for more than one blob to be sumbmitted via the cli blob, err := types.NewBlob(namespace, rawblob) if err != nil { return err diff --git a/x/blob/keeper/gas_test.go b/x/blob/keeper/gas_test.go index 32e6c50e47..f79f6327ae 100644 --- a/x/blob/keeper/gas_test.go +++ b/x/blob/keeper/gas_test.go @@ -3,6 +3,7 @@ package keeper import ( "testing" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" @@ -54,21 +55,33 @@ func TestPayForBlobGas(t *testing.T) { wantGasConsumed uint64 } + paramLookUpCost := uint32(1060) + testCases := []testCase{ { name: "1 byte blob", // occupies 1 share - msg: types.MsgPayForBlob{BlobSize: 1}, - wantGasConsumed: uint64(5156), // 1 share * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 5156 gas + msg: types.MsgPayForBlob{BlobSizes: []uint32{1}}, + wantGasConsumed: uint64(1*appconsts.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 1 share * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 5156 gas }, { name: "100 byte blob", // occupies 1 share - msg: types.MsgPayForBlob{BlobSize: 100}, - wantGasConsumed: uint64(5156), + msg: types.MsgPayForBlob{BlobSizes: []uint32{100}}, + wantGasConsumed: uint64(1*appconsts.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), }, { name: "1024 byte blob", // occupies 3 shares because share prefix (e.g. namespace, info byte) - msg: types.MsgPayForBlob{BlobSize: 1024}, - wantGasConsumed: uint64(13348), // 3 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 13348 gas + msg: types.MsgPayForBlob{BlobSizes: []uint32{1024}}, + wantGasConsumed: uint64(3*appconsts.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 3 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 13348 gas + }, + { + name: "3 blobs, 1 share each", + msg: types.MsgPayForBlob{BlobSizes: []uint32{1, 1, 1}}, + wantGasConsumed: uint64(3*appconsts.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 3 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 13348 gas + }, + { + name: "3 blobs, 6 shares total", + msg: types.MsgPayForBlob{BlobSizes: []uint32{1024, 1000, 100}}, + wantGasConsumed: uint64(6*appconsts.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 6 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 25636 gas }, } @@ -84,7 +97,7 @@ func TestPayForBlobGas(t *testing.T) { } func TestChangingGasParam(t *testing.T) { - msg := types.MsgPayForBlob{BlobSize: 1024} + msg := types.MsgPayForBlob{BlobSizes: []uint32{1024}} k, stateStore := keeper(t) tempCtx := sdk.NewContext(stateStore, tmproto.Header{}, false, nil) diff --git a/x/blob/keeper/keeper.go b/x/blob/keeper/keeper.go index ca03657d1c..3491160b6c 100644 --- a/x/blob/keeper/keeper.go +++ b/x/blob/keeper/keeper.go @@ -52,13 +52,16 @@ func (k Keeper) Logger(ctx sdk.Context) log.Logger { func (k Keeper) PayForBlob(goCtx context.Context, msg *types.MsgPayForBlob) (*types.MsgPayForBlobResponse, error) { ctx := sdk.UnwrapSDKContext(goCtx) - // calculate gas per blob share by fetching the constant share size and the gas cost per byte from the KV store - gasPerBlobShare := appconsts.ShareSize * k.GasPerBlobByte(ctx) - gasToConsume := uint64(shares.SparseSharesNeeded(uint32(msg.BlobSize)) * int(gasPerBlobShare)) - ctx.GasMeter().ConsumeGas(gasToConsume, payForBlobGasDescriptor) + totalSharesUsed := 0 + for _, size := range msg.BlobSizes { + totalSharesUsed += shares.SparseSharesNeeded(size) + } + + gasToConsume := uint32(totalSharesUsed*appconsts.ShareSize) * k.GasPerBlobByte(ctx) + ctx.GasMeter().ConsumeGas(uint64(gasToConsume), payForBlobGasDescriptor) err := ctx.EventManager().EmitTypedEvent( - types.NewPayForBlobEvent(sdk.AccAddress(msg.Signer).String(), msg.GetBlobSize(), msg.NamespaceId), + types.NewPayForBlobEvent(sdk.AccAddress(msg.Signer).String(), uint32(totalSharesUsed), msg.NamespaceIds), ) if err != nil { return &types.MsgPayForBlobResponse{}, err diff --git a/x/blob/payforblob.go b/x/blob/payforblob.go index 10c190ba2f..d7475c77d1 100644 --- a/x/blob/payforblob.go +++ b/x/blob/payforblob.go @@ -18,7 +18,7 @@ func SubmitPayForBlob( ctx context.Context, signer *types.KeyringSigner, conn *grpc.ClientConn, - blob *types.Blob, + blobs []*types.Blob, gasLim uint64, opts ...types.TxBuilderOption, ) (*sdk.TxResponse, error) { @@ -27,10 +27,7 @@ func SubmitPayForBlob( if err != nil { return nil, err } - msg, err := types.NewMsgPayForBlob( - addr.String(), - blob, - ) + msg, err := types.NewMsgPayForBlob(addr.String(), blobs...) if err != nil { return nil, err } @@ -47,7 +44,7 @@ func SubmitPayForBlob( if err != nil { return nil, err } - blobTx, err := coretypes.MarshalBlobTx(rawTx, blob) + blobTx, err := coretypes.MarshalBlobTx(rawTx, blobs...) if err != nil { return nil, err } diff --git a/x/blob/types/blob_tx.go b/x/blob/types/blob_tx.go index b1ae43ea5d..69615bc315 100644 --- a/x/blob/types/blob_tx.go +++ b/x/blob/types/blob_tx.go @@ -4,6 +4,7 @@ import ( "bytes" "github.com/celestiaorg/celestia-app/pkg/appconsts" + shares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/nmt/namespace" "github.com/cosmos/cosmos-sdk/client" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" @@ -50,33 +51,35 @@ func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx tmproto.BlobTx) error { if !ok { return ErrNoPFB } - // temporary check that we will remove when we support multiple blobs per PFB - if 1 != len(bTx.Blobs) { - return ErrMismatchedNumberOfPFBorBlob + err = pfb.ValidateBasic() + if err != nil { + return err } - // todo: modify this to support multiple messages per PFB - blob := bTx.Blobs[0] - err = pfb.ValidateBasic() + // perform basic checks on the blobs + sizes := make([]uint32, len(bTx.Blobs)) + for i, pblob := range bTx.Blobs { + sizes[i] = uint32(len(pblob.Data)) + } + err = ValidateBlobs(bTx.Blobs...) if err != nil { return err } - // check that the metadata matches - if !bytes.Equal(blob.NamespaceId, pfb.NamespaceId) { - return ErrNamespaceMismatch + // check that the info in the pfb matches that in the blobs + if !equalSlices(sizes, pfb.BlobSizes) { + return ErrBlobSizeMismatch.Wrapf("actual %v declared %v", sizes, pfb.BlobSizes) } - if pfb.BlobSize != uint32(len(blob.Data)) { - return ErrDeclaredActualDataSizeMismatch.Wrapf( - "declared: %d vs actual: %d", - pfb.BlobSize, - len(blob.Data), - ) + for i := range pfb.NamespaceIds { + // check that the metadata matches + if !bytes.Equal(bTx.Blobs[i].NamespaceId, pfb.NamespaceIds[i]) { + return ErrNamespaceMismatch.Wrapf("%v %v", bTx.Blobs[i].NamespaceId, pfb.NamespaceIds[i]) + } } // verify that the commitment of the blob matches that of the PFB - calculatedCommit, err := CreateMultiShareCommitment(blob) + calculatedCommit, err := CreateMultiShareCommitment(bTx.Blobs...) if err != nil { return ErrCalculateCommit } @@ -86,3 +89,23 @@ func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx tmproto.BlobTx) error { return nil } + +func BlobTxSharesUsed(btx tmproto.BlobTx) int { + sharesUsed := 0 + for _, blob := range btx.Blobs { + sharesUsed += shares.SparseSharesNeeded(uint32(len(blob.Data))) + } + return sharesUsed +} + +func equalSlices[T comparable](a, b []T) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/x/blob/types/blob_tx_test.go b/x/blob/types/blob_tx_test.go index 72fef3e9e6..bfebe7b4d4 100644 --- a/x/blob/types/blob_tx_test.go +++ b/x/blob/types/blob_tx_test.go @@ -46,6 +46,7 @@ func TestVerifySignature(t *testing.T) { } msg, blob := randMsgPayForBlobWithNamespaceAndSigner( + t, addr.String(), namespace.RandomBlobNamespace(), 100, @@ -57,10 +58,7 @@ func TestVerifySignature(t *testing.T) { rawTx, err := encCfg.TxConfig.TxEncoder()(stx) require.NoError(t, err) - wblob, err := NewBlob(msg.NamespaceId, blob) - require.NoError(t, err) - - cTx, err := coretypes.MarshalBlobTx(rawTx, wblob) + cTx, err := coretypes.MarshalBlobTx(rawTx, blob) require.NoError(t, err) uTx, isBlob := coretypes.UnmarshalBlobTx(cTx) @@ -106,15 +104,12 @@ func setupSigTest(t *testing.T) (string, sdk.Address, *KeyringSigner, encoding.C return acc, addr, signer, encCfg } -func randMsgPayForBlobWithNamespaceAndSigner(signer string, nid []byte, size int) (*MsgPayForBlob, []byte) { - blob := tmrand.Bytes(size) +func randMsgPayForBlobWithNamespaceAndSigner(t *testing.T, signer string, nid []byte, size int) (*MsgPayForBlob, *tmproto.Blob) { + blob, err := NewBlob(nid, tmrand.Bytes(size)) + require.NoError(t, err) msg, err := NewMsgPayForBlob( signer, - &tmproto.Blob{ - NamespaceId: nid, - Data: blob, - ShareVersion: uint32(appconsts.ShareVersionZero), - }, + blob, ) if err != nil { panic(err) diff --git a/x/blob/types/errors.go b/x/blob/types/errors.go index 1ad7d27df1..a6418bbfb6 100644 --- a/x/blob/types/errors.go +++ b/x/blob/types/errors.go @@ -10,7 +10,7 @@ var ( ErrReservedNamespace = sdkerrors.Register(ModuleName, 11110, "cannot use reserved namespace IDs") ErrInvalidNamespaceLen = sdkerrors.Register(ModuleName, 11111, "invalid namespace length") ErrInvalidDataSize = sdkerrors.Register(ModuleName, 11112, "data must be multiple of shareSize") - ErrDeclaredActualDataSizeMismatch = sdkerrors.Register(ModuleName, 11113, "declared data size does not match actual size") + ErrBlobSizeMismatch = sdkerrors.Register(ModuleName, 11113, "actual blob size differs from that specified in the MsgPayForBlob") ErrCommittedSquareSizeNotPowOf2 = sdkerrors.Register(ModuleName, 11114, "committed to invalid square size: must be power of two") ErrCalculateCommit = sdkerrors.Register(ModuleName, 11115, "unexpected error calculating commit for share") ErrInvalidShareCommit = sdkerrors.Register(ModuleName, 11116, "invalid commit for share") @@ -27,5 +27,6 @@ var ( ErrNamespaceMismatch = sdkerrors.Register(ModuleName, 11127, "namespace of blob and its respective MsgPayForBlob differ") ErrProtoParsing = sdkerrors.Register(ModuleName, 11128, "failure to parse a transaction from its protobuf representation") ErrMultipleMsgsInBlobTx = sdkerrors.Register(ModuleName, 11129, "not yet supported: multiple sdk.Msgs found in BlobTx") - ErrBloblessPFB = sdkerrors.Register(ModuleName, 11130, "transaction that contains a MsgPayForBlob must also contain a blob") + ErrMismatchedNumberOfPFBComponent = sdkerrors.Register(ModuleName, 11130, "number of each component in a MsgPayForBlob must be identical") + ErrNoBlobs = sdkerrors.Register(ModuleName, 11131, "no blobs provided") ) diff --git a/x/blob/types/event.pb.go b/x/blob/types/event.pb.go index e475e44bb3..7962b5e7a1 100644 --- a/x/blob/types/event.pb.go +++ b/x/blob/types/event.pb.go @@ -26,9 +26,9 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package // EventPayForBlob defines an event that is emitted after a pay for blob has // been processed. type EventPayForBlob struct { - Signer string `protobuf:"bytes,1,opt,name=signer,proto3" json:"signer,omitempty"` - BlobSize uint32 `protobuf:"varint,2,opt,name=blob_size,json=blobSize,proto3" json:"blob_size,omitempty"` - NamespaceId []byte `protobuf:"bytes,3,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + Signer string `protobuf:"bytes,1,opt,name=signer,proto3" json:"signer,omitempty"` + BlobSize uint32 `protobuf:"varint,2,opt,name=blob_size,json=blobSize,proto3" json:"blob_size,omitempty"` + NamespaceIds [][]byte `protobuf:"bytes,3,rep,name=namespace_ids,json=namespaceIds,proto3" json:"namespace_ids,omitempty"` } func (m *EventPayForBlob) Reset() { *m = EventPayForBlob{} } @@ -78,9 +78,9 @@ func (m *EventPayForBlob) GetBlobSize() uint32 { return 0 } -func (m *EventPayForBlob) GetNamespaceId() []byte { +func (m *EventPayForBlob) GetNamespaceIds() [][]byte { if m != nil { - return m.NamespaceId + return m.NamespaceIds } return nil } @@ -92,21 +92,21 @@ func init() { func init() { proto.RegisterFile("blob/event.proto", fileDescriptor_6c2ee49ac7fcb08f) } var fileDescriptor_6c2ee49ac7fcb08f = []byte{ - // 220 bytes of a gzipped FileDescriptorProto + // 222 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x48, 0xca, 0xc9, 0x4f, 0xd2, 0x4f, 0x2d, 0x4b, 0xcd, 0x2b, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x01, 0x89, - 0x48, 0x89, 0xa4, 0xe7, 0xa7, 0xe7, 0x83, 0x05, 0xf4, 0x41, 0x2c, 0x88, 0x9c, 0x52, 0x26, 0x17, + 0x48, 0x89, 0xa4, 0xe7, 0xa7, 0xe7, 0x83, 0x05, 0xf4, 0x41, 0x2c, 0x88, 0x9c, 0x52, 0x36, 0x17, 0xbf, 0x2b, 0x48, 0x69, 0x40, 0x62, 0xa5, 0x5b, 0x7e, 0x91, 0x53, 0x4e, 0x7e, 0x92, 0x90, 0x18, 0x17, 0x5b, 0x71, 0x66, 0x7a, 0x5e, 0x6a, 0x91, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x67, 0x10, 0x94, 0x27, 0x24, 0xcd, 0xc5, 0x09, 0x32, 0x28, 0xbe, 0x38, 0xb3, 0x2a, 0x55, 0x82, 0x49, 0x81, 0x51, - 0x83, 0x37, 0x88, 0x03, 0x24, 0x10, 0x9c, 0x59, 0x95, 0x2a, 0xa4, 0xc8, 0xc5, 0x93, 0x97, 0x98, - 0x9b, 0x5a, 0x5c, 0x90, 0x98, 0x9c, 0x1a, 0x9f, 0x99, 0x22, 0xc1, 0xac, 0xc0, 0xa8, 0xc1, 0x13, - 0xc4, 0x0d, 0x17, 0xf3, 0x4c, 0x71, 0xf2, 0x3a, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, - 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, 0x63, 0x39, - 0x86, 0x28, 0x83, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0xfd, 0xe4, 0xd4, - 0x9c, 0xd4, 0xe2, 0x92, 0xcc, 0xc4, 0xfc, 0xa2, 0x74, 0x38, 0x5b, 0x37, 0xb1, 0xa0, 0x40, 0xbf, - 0x42, 0x1f, 0xec, 0xb1, 0x92, 0xca, 0x82, 0xd4, 0xe2, 0x24, 0x36, 0xb0, 0xeb, 0x8d, 0x01, 0x01, - 0x00, 0x00, 0xff, 0xff, 0x2f, 0x11, 0xff, 0x2c, 0xed, 0x00, 0x00, 0x00, + 0x83, 0x37, 0x88, 0x03, 0x24, 0x10, 0x9c, 0x59, 0x95, 0x2a, 0xa4, 0xcc, 0xc5, 0x9b, 0x97, 0x98, + 0x9b, 0x5a, 0x5c, 0x90, 0x98, 0x9c, 0x1a, 0x9f, 0x99, 0x52, 0x2c, 0xc1, 0xac, 0xc0, 0xac, 0xc1, + 0x13, 0xc4, 0x03, 0x17, 0xf4, 0x4c, 0x29, 0x76, 0xf2, 0x3a, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, + 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, + 0x63, 0x39, 0x86, 0x28, 0x83, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0xfd, + 0xe4, 0xd4, 0x9c, 0xd4, 0xe2, 0x92, 0xcc, 0xc4, 0xfc, 0xa2, 0x74, 0x38, 0x5b, 0x37, 0xb1, 0xa0, + 0x40, 0xbf, 0x42, 0x1f, 0xec, 0xb5, 0x92, 0xca, 0x82, 0xd4, 0xe2, 0x24, 0x36, 0xb0, 0xfb, 0x8d, + 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xeb, 0x97, 0x8a, 0xfe, 0xef, 0x00, 0x00, 0x00, } func (m *EventPayForBlob) Marshal() (dAtA []byte, err error) { @@ -129,12 +129,14 @@ func (m *EventPayForBlob) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.NamespaceId) > 0 { - i -= len(m.NamespaceId) - copy(dAtA[i:], m.NamespaceId) - i = encodeVarintEvent(dAtA, i, uint64(len(m.NamespaceId))) - i-- - dAtA[i] = 0x1a + if len(m.NamespaceIds) > 0 { + for iNdEx := len(m.NamespaceIds) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.NamespaceIds[iNdEx]) + copy(dAtA[i:], m.NamespaceIds[iNdEx]) + i = encodeVarintEvent(dAtA, i, uint64(len(m.NamespaceIds[iNdEx]))) + i-- + dAtA[i] = 0x1a + } } if m.BlobSize != 0 { i = encodeVarintEvent(dAtA, i, uint64(m.BlobSize)) @@ -175,9 +177,11 @@ func (m *EventPayForBlob) Size() (n int) { if m.BlobSize != 0 { n += 1 + sovEvent(uint64(m.BlobSize)) } - l = len(m.NamespaceId) - if l > 0 { - n += 1 + l + sovEvent(uint64(l)) + if len(m.NamespaceIds) > 0 { + for _, b := range m.NamespaceIds { + l = len(b) + n += 1 + l + sovEvent(uint64(l)) + } } return n } @@ -270,7 +274,7 @@ func (m *EventPayForBlob) Unmarshal(dAtA []byte) error { } case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NamespaceId", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field NamespaceIds", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -297,10 +301,8 @@ func (m *EventPayForBlob) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NamespaceId = append(m.NamespaceId[:0], dAtA[iNdEx:postIndex]...) - if m.NamespaceId == nil { - m.NamespaceId = []byte{} - } + m.NamespaceIds = append(m.NamespaceIds, make([]byte, postIndex-iNdEx)) + copy(m.NamespaceIds[len(m.NamespaceIds)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex diff --git a/x/blob/types/events.go b/x/blob/types/events.go index 21144f5acd..7012212bfb 100644 --- a/x/blob/types/events.go +++ b/x/blob/types/events.go @@ -5,10 +5,10 @@ import "github.com/cosmos/gogoproto/proto" var EventTypePayForBlob = proto.MessageName(&EventPayForBlob{}) // NewPayForBlobEvent returns a new EventPayForBlob -func NewPayForBlobEvent(signer string, blobSize uint32, namespaceID []byte) *EventPayForBlob { +func NewPayForBlobEvent(signer string, blobSize uint32, namespaceIDs [][]byte) *EventPayForBlob { return &EventPayForBlob{ - Signer: signer, - BlobSize: blobSize, - NamespaceId: namespaceID, + Signer: signer, + BlobSize: blobSize, + NamespaceIds: namespaceIDs, } } diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 5aedab780a..1bd90c8a2a 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -11,8 +11,10 @@ import ( "github.com/celestiaorg/nmt/namespace" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/tendermint/tendermint/crypto/merkle" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" "golang.org/x/exp/constraints" + "golang.org/x/exp/slices" ) const ( @@ -25,20 +27,26 @@ const ( var _ sdk.Msg = &MsgPayForBlob{} -func NewMsgPayForBlob(signer string, blob *Blob) (*MsgPayForBlob, error) { - commitment, err := CreateMultiShareCommitment(blob) +func NewMsgPayForBlob(signer string, blobs ...*Blob) (*MsgPayForBlob, error) { + nsIDs, sizes, versions := extractBlobComponents(blobs) + err := ValidateBlobs(blobs...) if err != nil { return nil, err } - if len(blob.Data) == 0 { - return nil, ErrZeroBlobSize + + commitment, err := CreateMultiShareCommitment(blobs...) + if err != nil { + return nil, err } + msg := &MsgPayForBlob{ Signer: signer, - NamespaceId: blob.NamespaceId, + NamespaceIds: nsIDs, ShareCommitment: commitment, - BlobSize: uint32(len(blob.Data)), + BlobSizes: sizes, + ShareVersions: versions, } + return msg, msg.ValidateBasic() } @@ -53,8 +61,24 @@ func (msg *MsgPayForBlob) Type() string { // ValidateBasic fulfills the sdk.Msg interface by performing stateless // validity checks on the msg that also don't require having the actual blob func (msg *MsgPayForBlob) ValidateBasic() error { - if err := ValidateBlobNamespaceID(msg.GetNamespaceId()); err != nil { - return err + if len(msg.NamespaceIds) != len(msg.ShareVersions) || len(msg.NamespaceIds) != len(msg.BlobSizes) { + return ErrMismatchedNumberOfPFBComponent.Wrapf( + "namespaces %d blob sizes %d versions %d", + len(msg.NamespaceIds), len(msg.BlobSizes), len(msg.ShareVersions), + ) + } + + for _, ns := range msg.NamespaceIds { + err := ValidateBlobNamespaceID(ns) + if err != nil { + return err + } + } + + for _, v := range msg.ShareVersions { + if v != uint32(appconsts.ShareVersionZero) { + return ErrUnsupportedShareVersion + } } _, err := sdk.AccAddressFromBech32(msg.Signer) @@ -156,6 +180,30 @@ func CreateMultiShareCommitment(blobs ...*Blob) ([]byte, error) { return merkle.HashFromByteSlices(commitments), nil } +// ValidatePFBComponents performs basic checks over the components of one or more PFBs. +func ValidateBlobs(blobs ...*Blob) error { + if len(blobs) == 0 { + return ErrNoBlobs + } + + for _, blob := range blobs { + err := ValidateBlobNamespaceID(blob.NamespaceId) + if err != nil { + return err + } + + if len(blob.Data) == 0 { + return ErrZeroBlobSize + } + + if !slices.Contains(appconsts.SupportedShareVersions, uint8(blob.ShareVersion)) { + return ErrUnsupportedShareVersion + } + } + + return nil +} + // ValidateBlobNamespaceID returns an error if the provided namespace.ID is an invalid or reserved namespace id. func ValidateBlobNamespaceID(ns namespace.ID) error { // ensure that the namespace id is of length == NamespaceIDSize @@ -183,6 +231,23 @@ func ValidateBlobNamespaceID(ns namespace.ID) error { return nil } +// extractBlobComponents separates and returns the components of a slice of +// blobs in order of blobs of data, their namespaces, their sizes, and their share +// versions. +func extractBlobComponents(pblobs []*tmproto.Blob) (nsIDs [][]byte, sizes []uint32, versions []uint32) { + nsIDs = make([][]byte, len(pblobs)) + sizes = make([]uint32, len(pblobs)) + versions = make([]uint32, len(pblobs)) + + for i, pblob := range pblobs { + sizes[i] = uint32(len(pblob.Data)) + nsIDs[i] = pblob.NamespaceId + versions[i] = pblob.ShareVersion + } + + return nsIDs, sizes, versions +} + // BlobMinSquareSize returns the minimum square size that blobSize can be included // in. The returned square size does not account for the associated transaction // shares or non-interactive defaults, so it is a minimum. diff --git a/x/blob/types/payforblob_test.go b/x/blob/types/payforblob_test.go index d570dda9b3..d5e1af7c17 100644 --- a/x/blob/types/payforblob_test.go +++ b/x/blob/types/payforblob_test.go @@ -121,27 +121,27 @@ func TestValidateBasic(t *testing.T) { // MsgPayForBlob that uses parity shares namespace id paritySharesMsg := validMsgPayForBlob(t) - paritySharesMsg.NamespaceId = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} + paritySharesMsg.NamespaceIds[0] = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} // MsgPayForBlob that uses tail padding namespace id tailPaddingMsg := validMsgPayForBlob(t) - tailPaddingMsg.NamespaceId = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE} + tailPaddingMsg.NamespaceIds[0] = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE} // MsgPayForBlob that uses transaction namespace id txNamespaceMsg := validMsgPayForBlob(t) - txNamespaceMsg.NamespaceId = namespace.ID{0, 0, 0, 0, 0, 0, 0, 1} + txNamespaceMsg.NamespaceIds[0] = namespace.ID{0, 0, 0, 0, 0, 0, 0, 1} // MsgPayForBlob that uses intermediateStateRoots namespace id intermediateStateRootsNamespaceMsg := validMsgPayForBlob(t) - intermediateStateRootsNamespaceMsg.NamespaceId = namespace.ID{0, 0, 0, 0, 0, 0, 0, 2} + intermediateStateRootsNamespaceMsg.NamespaceIds[0] = namespace.ID{0, 0, 0, 0, 0, 0, 0, 2} // MsgPayForBlob that uses evidence namespace id evidenceNamespaceMsg := validMsgPayForBlob(t) - evidenceNamespaceMsg.NamespaceId = namespace.ID{0, 0, 0, 0, 0, 0, 0, 3} + evidenceNamespaceMsg.NamespaceIds[0] = namespace.ID{0, 0, 0, 0, 0, 0, 0, 3} // MsgPayForBlob that uses the max reserved namespace id maxReservedNamespaceMsg := validMsgPayForBlob(t) - maxReservedNamespaceMsg.NamespaceId = namespace.ID{0, 0, 0, 0, 0, 0, 0, 255} + maxReservedNamespaceMsg.NamespaceIds[0] = namespace.ID{0, 0, 0, 0, 0, 0, 0, 255} // MsgPayForBlob that has an empty share commitment emptyShareCommitment := validMsgPayForBlob(t) @@ -218,14 +218,13 @@ func validMsgPayForBlob(t *testing.T) *MsgPayForBlob { addr, err := signer.GetSignerInfo().GetAddress() require.NoError(t, err) - pfb, err := NewMsgPayForBlob( - addr.String(), - &tmproto.Blob{ - NamespaceId: ns, - Data: blob, - ShareVersion: uint32(appconsts.ShareVersionZero), - }, - ) + pblob := &tmproto.Blob{ + Data: blob, + NamespaceId: ns, + ShareVersion: uint32(appconsts.ShareVersionZero), + } + + pfb, err := NewMsgPayForBlob(addr.String(), pblob) assert.NoError(t, err) return pfb @@ -294,8 +293,6 @@ func TestNewMsgPayForBlob(t *testing.T) { expectedCommitment, err := CreateMultiShareCommitment(blob) require.NoError(t, err) assert.Equal(t, expectedCommitment, res.ShareCommitment) - - assert.Equal(t, uint32(len(tt.blobs[0])), res.BlobSize) } } @@ -339,3 +336,27 @@ func TestBlobMinSquareSize(t *testing.T) { }) } } + +func TestValidateBlobs(t *testing.T) { + type test struct { + name string + blob *Blob + expectError bool + } + + tests := []test{ + {name: "valid blob", blob: &Blob{Data: []byte{1}, NamespaceId: []byte{1, 2, 3, 4, 5, 6, 7, 8}, ShareVersion: uint32(appconsts.DefaultShareVersion)}, expectError: false}, + {name: "invalid share version", blob: &Blob{Data: []byte{1}, NamespaceId: []byte{1, 2, 3, 4, 5, 6, 7, 8}, ShareVersion: uint32(10000)}, expectError: true}, + {name: "empty blob", blob: &Blob{Data: []byte{}, NamespaceId: []byte{1, 2, 3, 4, 5, 6, 7, 8}, ShareVersion: uint32(appconsts.DefaultShareVersion)}, expectError: true}, + {name: "invalid namespace", blob: &Blob{Data: []byte{1}, NamespaceId: appconsts.TxNamespaceID, ShareVersion: uint32(appconsts.DefaultShareVersion)}, expectError: true}, + } + + for _, tt := range tests { + err := ValidateBlobs(tt.blob) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + } +} diff --git a/x/blob/types/test/blob_tx_test.go b/x/blob/types/test/blob_tx_test.go index 9583822f53..367ba02023 100644 --- a/x/blob/types/test/blob_tx_test.go +++ b/x/blob/types/test/blob_tx_test.go @@ -58,7 +58,7 @@ func TestValidateBlobTx(t *testing.T) { getTx: func() tmproto.BlobTx { rawBtx := validRawBtx() btx, _ := coretypes.UnmarshalBlobTx(rawBtx) - btx.Blobs[0].NamespaceId = appconsts.TxNamespaceID + btx.Blobs[0].NamespaceId = namespace.RandomBlobNamespace() return btx }, expectedErr: types.ErrNamespaceMismatch, @@ -86,15 +86,16 @@ func TestValidateBlobTx(t *testing.T) { btx.Blobs = append(btx.Blobs, blob) return btx }, - expectedErr: types.ErrMismatchedNumberOfPFBorBlob, + expectedErr: types.ErrBlobSizeMismatch, }, { name: "invalid share commitment", getTx: func() tmproto.BlobTx { - rawblob := rand.Bytes(100) + blob, err := types.NewBlob(namespace.RandomBlobNamespace(), rand.Bytes(100)) + require.NoError(t, err) msg, err := types.NewMsgPayForBlob( signerAddr.String(), - &tmproto.Blob{NamespaceId: namespace.RandomBlobNamespace(), Data: rawblob, ShareVersion: 0}, + blob, ) require.NoError(t, err) @@ -114,11 +115,9 @@ func TestValidateBlobTx(t *testing.T) { rawTx, err := encCfg.TxConfig.TxEncoder()(stx) require.NoError(t, err) - wblob, err := types.NewBlob(msg.NamespaceId, rawblob) - require.NoError(t, err) btx := tmproto.BlobTx{ Tx: rawTx, - Blobs: []*tmproto.Blob{wblob}, + Blobs: []*tmproto.Blob{blob}, } return btx }, @@ -155,6 +154,83 @@ func TestValidateBlobTx(t *testing.T) { }, expectedErr: types.ErrNoPFB, }, + { + name: "normal transaction with two blobs w/ different namespaces", + getTx: func() tmproto.BlobTx { + rawBtx := blobfactory.MultiBlobTx( + t, + encCfg.TxConfig.TxEncoder(), + signer, + blobfactory.RandBlobsWithNamespace( + [][]byte{namespace.RandomBlobNamespace(), namespace.RandomBlobNamespace()}, + []int{100, 100})..., + ) + btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + require.True(t, isBlobTx) + return btx + }, + expectedErr: nil, + }, + { + name: "normal transaction with two large blobs w/ different namespaces", + getTx: func() tmproto.BlobTx { + rawBtx := blobfactory.MultiBlobTx( + t, + encCfg.TxConfig.TxEncoder(), + signer, + blobfactory.RandBlobsWithNamespace( + [][]byte{namespace.RandomBlobNamespace(), namespace.RandomBlobNamespace()}, + []int{100000, 1000000})..., + ) + btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + require.True(t, isBlobTx) + return btx + }, + expectedErr: nil, + }, + { + name: "normal transaction with two blobs w/ same namespace", + getTx: func() tmproto.BlobTx { + ns := namespace.RandomBlobNamespace() + rawBtx := blobfactory.MultiBlobTx( + t, + encCfg.TxConfig.TxEncoder(), + signer, + blobfactory.RandBlobsWithNamespace( + [][]byte{ns, ns}, + []int{100, 100})..., + ) + btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + require.True(t, isBlobTx) + return btx + }, + expectedErr: nil, + }, + { + name: "normal transaction with one hundred blobs of the same namespace", + getTx: func() tmproto.BlobTx { + count := 100 + ns := namespace.RandomBlobNamespace() + sizes := make([]int, count) + namespaces := make([][]byte, count) + for i := 0; i < count; i++ { + sizes[i] = 100 + namespaces[i] = ns + } + rawBtx := blobfactory.MultiBlobTx( + t, + encCfg.TxConfig.TxEncoder(), + signer, + blobfactory.RandBlobsWithNamespace( + namespaces, + sizes, + )...) + btx, isBlobTx := coretypes.UnmarshalBlobTx(rawBtx) + require.True(t, isBlobTx) + return btx + }, + expectedErr: nil, + }, } for _, tt := range tests { diff --git a/x/blob/types/tx.pb.go b/x/blob/types/tx.pb.go index f64528f9aa..478919611d 100644 --- a/x/blob/types/tx.pb.go +++ b/x/blob/types/tx.pb.go @@ -87,17 +87,17 @@ func (m *ShareCommitAndSignature) GetSignature() []byte { // MsgPayForBlob pays for the inclusion of a blob in the block. type MsgPayForBlob struct { - Signer string `protobuf:"bytes,1,opt,name=signer,proto3" json:"signer,omitempty"` - NamespaceId []byte `protobuf:"bytes,2,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` - BlobSize uint32 `protobuf:"varint,3,opt,name=blob_size,json=blobSize,proto3" json:"blob_size,omitempty"` + Signer string `protobuf:"bytes,1,opt,name=signer,proto3" json:"signer,omitempty"` + NamespaceIds [][]byte `protobuf:"bytes,2,rep,name=namespace_ids,json=namespaceIds,proto3" json:"namespace_ids,omitempty"` + BlobSizes []uint32 `protobuf:"varint,3,rep,packed,name=blob_sizes,json=blobSizes,proto3" json:"blob_sizes,omitempty"` // share_commitment is the share_commitment from // ShareCommitAndSignature that will be included in a block ShareCommitment []byte `protobuf:"bytes,4,opt,name=share_commitment,json=shareCommitment,proto3" json:"share_commitment,omitempty"` - // share_version is the version of the share format that the blob associated - // with this message should use when included in a block. The share_version - // specified must match the share_version used to generate the + // share_versions are the versions of the share format that the blobs + // associated with this message should use when included in a block. The + // share_versions specified must match the share_versions used to generate the // share_commitment in this message. - ShareVersion uint32 `protobuf:"varint,8,opt,name=share_version,json=shareVersion,proto3" json:"share_version,omitempty"` + ShareVersions []uint32 `protobuf:"varint,8,rep,packed,name=share_versions,json=shareVersions,proto3" json:"share_versions,omitempty"` } func (m *MsgPayForBlob) Reset() { *m = MsgPayForBlob{} } @@ -140,18 +140,18 @@ func (m *MsgPayForBlob) GetSigner() string { return "" } -func (m *MsgPayForBlob) GetNamespaceId() []byte { +func (m *MsgPayForBlob) GetNamespaceIds() [][]byte { if m != nil { - return m.NamespaceId + return m.NamespaceIds } return nil } -func (m *MsgPayForBlob) GetBlobSize() uint32 { +func (m *MsgPayForBlob) GetBlobSizes() []uint32 { if m != nil { - return m.BlobSize + return m.BlobSizes } - return 0 + return nil } func (m *MsgPayForBlob) GetShareCommitment() []byte { @@ -161,11 +161,11 @@ func (m *MsgPayForBlob) GetShareCommitment() []byte { return nil } -func (m *MsgPayForBlob) GetShareVersion() uint32 { +func (m *MsgPayForBlob) GetShareVersions() []uint32 { if m != nil { - return m.ShareVersion + return m.ShareVersions } - return 0 + return nil } // MsgPayForBlobResponse describes the response returned after the submission @@ -215,31 +215,32 @@ func init() { func init() { proto.RegisterFile("blob/tx.proto", fileDescriptor_f945cb94fe124aae) } var fileDescriptor_f945cb94fe124aae = []byte{ - // 381 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcf, 0xaa, 0xda, 0x40, - 0x14, 0xc6, 0x8d, 0x8a, 0xe8, 0xd4, 0xd0, 0x32, 0xfd, 0x63, 0xaa, 0x12, 0x34, 0xdd, 0xd8, 0x45, - 0x93, 0xd2, 0x3e, 0x41, 0x2d, 0x14, 0x5a, 0x10, 0x4a, 0x84, 0x2e, 0xba, 0x91, 0x49, 0x9c, 0x8e, - 0x03, 0xc9, 0x9c, 0x61, 0x66, 0x2c, 0xea, 0xb2, 0x4f, 0x50, 0xe8, 0xc3, 0xf4, 0x15, 0xba, 0x14, - 0xba, 0xb9, 0xcb, 0x8b, 0xde, 0x07, 0xb9, 0x4c, 0xe2, 0x9f, 0x2b, 0xb8, 0x3b, 0xf3, 0xfb, 0xe6, - 0x3b, 0xe7, 0x3b, 0x1c, 0xe4, 0x26, 0x19, 0x24, 0x91, 0x59, 0x85, 0x52, 0x81, 0x01, 0x5c, 0xb7, - 0xcf, 0xee, 0x33, 0x06, 0x0c, 0x0a, 0x10, 0xd9, 0xaa, 0xd4, 0xba, 0x7d, 0x06, 0xc0, 0x32, 0x1a, - 0x11, 0xc9, 0x23, 0x22, 0x04, 0x18, 0x62, 0x38, 0x08, 0x5d, 0xaa, 0x41, 0x82, 0x3a, 0xd3, 0x05, - 0x51, 0xf4, 0x23, 0xe4, 0x39, 0x37, 0x1f, 0xc4, 0x7c, 0xca, 0x99, 0x20, 0x66, 0xa9, 0x28, 0x7e, - 0x8d, 0x9e, 0x68, 0x2b, 0xcd, 0xd2, 0x42, 0xcb, 0xa9, 0x30, 0x5e, 0x75, 0xe0, 0x8c, 0xda, 0xf1, - 0x63, 0x7d, 0xb6, 0x58, 0x8c, 0xfb, 0xa8, 0xa5, 0x8f, 0x3e, 0xaf, 0x56, 0xfc, 0x39, 0x83, 0xe0, - 0xaf, 0x83, 0xdc, 0x89, 0x66, 0x5f, 0xc9, 0xfa, 0x13, 0xa8, 0x71, 0x06, 0x09, 0x7e, 0x81, 0x1a, - 0x56, 0xa6, 0xca, 0x73, 0x06, 0xce, 0xa8, 0x15, 0x1f, 0x5e, 0x78, 0x88, 0xda, 0x82, 0xe4, 0x54, - 0x4b, 0x92, 0xd2, 0x19, 0x9f, 0x1f, 0xc6, 0x3d, 0x3a, 0xb1, 0xcf, 0x73, 0xdc, 0x43, 0x2d, 0xbb, - 0xec, 0x4c, 0xf3, 0x4d, 0x39, 0xca, 0x8d, 0x9b, 0x16, 0x4c, 0xf9, 0xe6, 0x7a, 0xe4, 0xfa, 0xf5, - 0xc8, 0xaf, 0x90, 0x5b, 0x7e, 0xfd, 0x49, 0x95, 0xe6, 0x20, 0xbc, 0x66, 0xd1, 0xab, 0x5d, 0xc0, - 0x6f, 0x25, 0x0b, 0x3a, 0xe8, 0xf9, 0x45, 0xf0, 0x98, 0x6a, 0x09, 0x42, 0xd3, 0x77, 0x0b, 0x54, - 0x9b, 0x68, 0x86, 0x09, 0x42, 0x0f, 0xb6, 0x7a, 0x1a, 0xda, 0x20, 0xe1, 0x85, 0xa3, 0xdb, 0xbb, - 0x02, 0x8f, 0x6d, 0x82, 0xe1, 0xaf, 0xff, 0x77, 0x7f, 0xaa, 0x3d, 0xfc, 0x32, 0x4a, 0x69, 0x46, - 0xb5, 0xe1, 0x24, 0x2a, 0x0e, 0x2b, 0xc9, 0xfa, 0x07, 0x28, 0x5b, 0x8e, 0xbf, 0xfc, 0xdb, 0xf9, - 0xce, 0x76, 0xe7, 0x3b, 0xb7, 0x3b, 0xdf, 0xf9, 0xbd, 0xf7, 0x2b, 0xdb, 0xbd, 0x5f, 0xb9, 0xd9, - 0xfb, 0x95, 0xef, 0x6f, 0x19, 0x37, 0x8b, 0x65, 0x12, 0xa6, 0x90, 0x9f, 0xec, 0xa0, 0xd8, 0xa9, - 0x7e, 0x43, 0xa4, 0x8c, 0x56, 0x65, 0x43, 0xb3, 0x96, 0x54, 0x27, 0x8d, 0xe2, 0xe6, 0xef, 0xef, - 0x03, 0x00, 0x00, 0xff, 0xff, 0x3a, 0x1f, 0x3a, 0x3e, 0x3e, 0x02, 0x00, 0x00, + // 389 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcf, 0xaa, 0xd3, 0x40, + 0x14, 0xc6, 0x9b, 0xe6, 0x72, 0xb1, 0x43, 0xa3, 0x32, 0xfe, 0xb9, 0xb1, 0xf7, 0x1a, 0x62, 0x44, + 0x88, 0x0b, 0x13, 0xd1, 0x27, 0xf0, 0x0a, 0x82, 0xc2, 0x05, 0x49, 0xc1, 0x85, 0x9b, 0x32, 0x49, + 0xc7, 0xe9, 0x40, 0x32, 0x67, 0x98, 0x33, 0x57, 0x5a, 0x97, 0x3e, 0x81, 0xe0, 0xf3, 0xb8, 0x77, + 0x59, 0x70, 0xe3, 0x52, 0x5a, 0x1f, 0x44, 0x26, 0xe9, 0x1f, 0x0b, 0xdd, 0x9d, 0xf9, 0x7d, 0xe7, + 0x3b, 0x7c, 0x67, 0x0e, 0x09, 0xca, 0x1a, 0xca, 0xdc, 0xce, 0x33, 0x6d, 0xc0, 0x02, 0x3d, 0x71, + 0xcf, 0xd1, 0x5d, 0x01, 0x02, 0x5a, 0x90, 0xbb, 0xaa, 0xd3, 0x46, 0x17, 0x02, 0x40, 0xd4, 0x3c, + 0x67, 0x5a, 0xe6, 0x4c, 0x29, 0xb0, 0xcc, 0x4a, 0x50, 0xd8, 0xa9, 0x49, 0x49, 0xce, 0xc6, 0x33, + 0x66, 0xf8, 0x6b, 0x68, 0x1a, 0x69, 0x5f, 0xa9, 0xe9, 0x58, 0x0a, 0xc5, 0xec, 0xb5, 0xe1, 0xf4, + 0x29, 0xb9, 0x8d, 0x4e, 0x9a, 0x54, 0xad, 0xd6, 0x70, 0x65, 0xc3, 0x7e, 0xec, 0xa5, 0xc3, 0xe2, + 0x16, 0xee, 0x2d, 0x0e, 0xd3, 0x0b, 0x32, 0xc0, 0xad, 0x2f, 0xf4, 0xdb, 0x9e, 0x3d, 0x48, 0x7e, + 0x78, 0x24, 0xb8, 0x42, 0xf1, 0x9e, 0x2d, 0xde, 0x80, 0xb9, 0xac, 0xa1, 0xa4, 0xf7, 0xc9, 0xa9, + 0x93, 0xb9, 0x09, 0xbd, 0xd8, 0x4b, 0x07, 0xc5, 0xe6, 0x45, 0x1f, 0x93, 0x40, 0xb1, 0x86, 0xa3, + 0x66, 0x15, 0x9f, 0xc8, 0x29, 0x86, 0xfd, 0xd8, 0x4f, 0x87, 0xc5, 0x70, 0x07, 0xdf, 0x4e, 0x91, + 0x3e, 0x24, 0xc4, 0xad, 0x3b, 0x41, 0xf9, 0x85, 0x63, 0xe8, 0xc7, 0x7e, 0x1a, 0x14, 0x03, 0x47, + 0xc6, 0x0e, 0x1c, 0x8d, 0x7d, 0x72, 0x3c, 0xf6, 0x13, 0x72, 0xb3, 0x6b, 0xfd, 0xcc, 0x0d, 0xba, + 0x4f, 0x09, 0x6f, 0xb4, 0xd3, 0x82, 0x96, 0x7e, 0xd8, 0xc0, 0xe4, 0x8c, 0xdc, 0x3b, 0x88, 0x5f, + 0x70, 0xd4, 0xa0, 0x90, 0xbf, 0x98, 0x11, 0xff, 0x0a, 0x05, 0x65, 0x84, 0xfc, 0xb7, 0xdb, 0x9d, + 0xcc, 0x65, 0xc9, 0x0e, 0x1c, 0xa3, 0xf3, 0x23, 0x70, 0x3b, 0x26, 0x79, 0xf4, 0xf5, 0xd7, 0xdf, + 0xef, 0xfd, 0x73, 0xfa, 0x20, 0xaf, 0x78, 0xcd, 0xd1, 0x4a, 0x96, 0xb7, 0xe7, 0xd5, 0x6c, 0xf1, + 0x09, 0x8c, 0x2b, 0x2f, 0xdf, 0xfd, 0x5c, 0x45, 0xde, 0x72, 0x15, 0x79, 0x7f, 0x56, 0x91, 0xf7, + 0x6d, 0x1d, 0xf5, 0x96, 0xeb, 0xa8, 0xf7, 0x7b, 0x1d, 0xf5, 0x3e, 0x3e, 0x17, 0xd2, 0xce, 0xae, + 0xcb, 0xac, 0x82, 0x66, 0x67, 0x07, 0x23, 0x76, 0xf5, 0x33, 0xa6, 0x75, 0x3e, 0xef, 0x06, 0xda, + 0x85, 0xe6, 0x58, 0x9e, 0xb6, 0x97, 0x7f, 0xf9, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xbf, 0x03, 0xf2, + 0x1c, 0x44, 0x02, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -381,10 +382,23 @@ func (m *MsgPayForBlob) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.ShareVersion != 0 { - i = encodeVarintTx(dAtA, i, uint64(m.ShareVersion)) + if len(m.ShareVersions) > 0 { + dAtA2 := make([]byte, len(m.ShareVersions)*10) + var j1 int + for _, num := range m.ShareVersions { + for num >= 1<<7 { + dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j1++ + } + dAtA2[j1] = uint8(num) + j1++ + } + i -= j1 + copy(dAtA[i:], dAtA2[:j1]) + i = encodeVarintTx(dAtA, i, uint64(j1)) i-- - dAtA[i] = 0x40 + dAtA[i] = 0x42 } if len(m.ShareCommitment) > 0 { i -= len(m.ShareCommitment) @@ -393,17 +407,32 @@ func (m *MsgPayForBlob) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x22 } - if m.BlobSize != 0 { - i = encodeVarintTx(dAtA, i, uint64(m.BlobSize)) + if len(m.BlobSizes) > 0 { + dAtA4 := make([]byte, len(m.BlobSizes)*10) + var j3 int + for _, num := range m.BlobSizes { + for num >= 1<<7 { + dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j3++ + } + dAtA4[j3] = uint8(num) + j3++ + } + i -= j3 + copy(dAtA[i:], dAtA4[:j3]) + i = encodeVarintTx(dAtA, i, uint64(j3)) i-- - dAtA[i] = 0x18 + dAtA[i] = 0x1a } - if len(m.NamespaceId) > 0 { - i -= len(m.NamespaceId) - copy(dAtA[i:], m.NamespaceId) - i = encodeVarintTx(dAtA, i, uint64(len(m.NamespaceId))) - i-- - dAtA[i] = 0x12 + if len(m.NamespaceIds) > 0 { + for iNdEx := len(m.NamespaceIds) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.NamespaceIds[iNdEx]) + copy(dAtA[i:], m.NamespaceIds[iNdEx]) + i = encodeVarintTx(dAtA, i, uint64(len(m.NamespaceIds[iNdEx]))) + i-- + dAtA[i] = 0x12 + } } if len(m.Signer) > 0 { i -= len(m.Signer) @@ -476,19 +505,29 @@ func (m *MsgPayForBlob) Size() (n int) { if l > 0 { n += 1 + l + sovTx(uint64(l)) } - l = len(m.NamespaceId) - if l > 0 { - n += 1 + l + sovTx(uint64(l)) + if len(m.NamespaceIds) > 0 { + for _, b := range m.NamespaceIds { + l = len(b) + n += 1 + l + sovTx(uint64(l)) + } } - if m.BlobSize != 0 { - n += 1 + sovTx(uint64(m.BlobSize)) + if len(m.BlobSizes) > 0 { + l = 0 + for _, e := range m.BlobSizes { + l += sovTx(uint64(e)) + } + n += 1 + sovTx(uint64(l)) + l } l = len(m.ShareCommitment) if l > 0 { n += 1 + l + sovTx(uint64(l)) } - if m.ShareVersion != 0 { - n += 1 + sovTx(uint64(m.ShareVersion)) + if len(m.ShareVersions) > 0 { + l = 0 + for _, e := range m.ShareVersions { + l += sovTx(uint64(e)) + } + n += 1 + sovTx(uint64(l)) + l } return n } @@ -689,7 +728,7 @@ func (m *MsgPayForBlob) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NamespaceId", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field NamespaceIds", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -716,29 +755,84 @@ func (m *MsgPayForBlob) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NamespaceId = append(m.NamespaceId[:0], dAtA[iNdEx:postIndex]...) - if m.NamespaceId == nil { - m.NamespaceId = []byte{} - } + m.NamespaceIds = append(m.NamespaceIds, make([]byte, postIndex-iNdEx)) + copy(m.NamespaceIds[len(m.NamespaceIds)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field BlobSize", wireType) - } - m.BlobSize = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTx + if wireType == 0 { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + m.BlobSizes = append(m.BlobSizes, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthTx + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthTx + } + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - m.BlobSize |= uint32(b&0x7F) << shift - if b < 0x80 { - break + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } } + elementCount = count + if elementCount != 0 && len(m.BlobSizes) == 0 { + m.BlobSizes = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.BlobSizes = append(m.BlobSizes, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BlobSizes", wireType) } case 4: if wireType != 2 { @@ -775,23 +869,80 @@ func (m *MsgPayForBlob) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 8: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ShareVersion", wireType) - } - m.ShareVersion = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTx + if wireType == 0 { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { + m.ShareVersions = append(m.ShareVersions, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthTx + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthTx + } + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - m.ShareVersion |= uint32(b&0x7F) << shift - if b < 0x80 { - break + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.ShareVersions) == 0 { + m.ShareVersions = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ShareVersions = append(m.ShareVersions, v) } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ShareVersions", wireType) } default: iNdEx = preIndex