Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge backported PRs into v1.10 release branch #6436

Merged
merged 4 commits into from
Jun 9, 2021
Merged

Merge backported PRs into v1.10 release branch #6436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f0a2e97
fee config for sector batching
magik6k Jun 8, 2021
0cd0faa
Increase message size limit
Jun 8, 2021
1804381
Create MaxMessageSize constant
Jun 8, 2021
626d482
correct the change of message size limit
GFZRZK Jun 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion chain/messagepool/messagepool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ var MaxUntrustedActorPendingMessages = 10

var MaxNonceGap = uint64(4)

const MaxMessageSize = 64 << 10 // 64KiB

var (
ErrMessageTooBig = errors.New("message too big")

Expand Down Expand Up @@ -665,7 +667,7 @@ func (mp *MessagePool) Push(ctx context.Context, m *types.SignedMessage) (cid.Ci

func (mp *MessagePool) checkMessage(m *types.SignedMessage) error {
// big messages are bad, anti DOS
if m.Size() > 32*1024 {
if m.Size() > MaxMessageSize {
return xerrors.Errorf("mpool message too large (%dB): %w", m.Size(), ErrMessageTooBig)
}

Expand Down
68 changes: 68 additions & 0 deletions chain/messagepool/messagepool_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ import (

builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"

"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/messagepool/gasguess"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/mock"
"github.com/filecoin-project/lotus/chain/wallet"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"
"github.com/stretchr/testify/assert"
)

func init() {
Expand Down Expand Up @@ -257,6 +259,72 @@ func TestMessagePool(t *testing.T) {
assertNonce(t, mp, sender, 2)
}

func TestCheckMessageBig(t *testing.T) {
tma := newTestMpoolAPI()

w, err := wallet.NewWallet(wallet.NewMemKeyStore())
assert.NoError(t, err)

from, err := w.WalletNew(context.Background(), types.KTBLS)
assert.NoError(t, err)

tma.setBalance(from, 1000e9)

ds := datastore.NewMapDatastore()

mp, err := New(tma, ds, "mptest", nil)
assert.NoError(t, err)

to := mock.Address(1001)

{
msg := &types.Message{
To: to,
From: from,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(100),
GasPremium: types.NewInt(1),
Params: make([]byte, 41<<10), // 41KiB payload
}

sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
if err != nil {
panic(err)
}
sm := &types.SignedMessage{
Message: *msg,
Signature: *sig,
}
mustAdd(t, mp, sm)
}

{
msg := &types.Message{
To: to,
From: from,
Value: types.NewInt(1),
Nonce: 0,
GasLimit: 50000000,
GasFeeCap: types.NewInt(100),
GasPremium: types.NewInt(1),
Params: make([]byte, 64<<10), // 64KiB payload
}

sig, err := w.WalletSign(context.TODO(), from, msg.Cid().Bytes(), api.MsgMeta{})
if err != nil {
panic(err)
}
sm := &types.SignedMessage{
Message: *msg,
Signature: *sig,
}
err = mp.Add(context.TODO(), sm)
assert.ErrorIs(t, err, ErrMessageTooBig)
}
}

func TestMessagePoolMessagesInEachBlock(t *testing.T) {
tma := newTestMpoolAPI()

Expand Down
2 changes: 1 addition & 1 deletion chain/sub/incoming.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,7 @@ func (mv *MessageValidator) validateLocalMessage(ctx context.Context, msg *pubsu
return pubsub.ValidationIgnore
}

if m.Size() > 32*1024 {
if m.Size() > messagepool.MaxMessageSize {
log.Warnf("local message is too large! (%dB)", m.Size())
recordFailure(ctx, metrics.MessageValidationFailure, "oversize")
return pubsub.ValidationIgnore
Expand Down
15 changes: 9 additions & 6 deletions extern/storage-sealing/commit_batch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
"github.com/filecoin-project/lotus/node/config"
)

const arp = abi.RegisteredAggregationProof_SnarkPackV1
Expand All @@ -53,7 +54,7 @@ type CommitBatcher struct {
maddr address.Address
mctx context.Context
addrSel AddrSel
feeCfg FeeConfig
feeCfg config.MinerFeeConfig
getConfig GetSealingConfigFunc
prover ffiwrapper.Prover

Expand All @@ -66,7 +67,7 @@ type CommitBatcher struct {
lk sync.Mutex
}

func NewCommitBatcher(mctx context.Context, maddr address.Address, api CommitBatcherApi, addrSel AddrSel, feeCfg FeeConfig, getConfig GetSealingConfigFunc, prov ffiwrapper.Prover) *CommitBatcher {
func NewCommitBatcher(mctx context.Context, maddr address.Address, api CommitBatcherApi, addrSel AddrSel, feeCfg config.MinerFeeConfig, getConfig GetSealingConfigFunc, prov ffiwrapper.Prover) *CommitBatcher {
b := &CommitBatcher{
api: api,
maddr: maddr,
Expand Down Expand Up @@ -291,6 +292,8 @@ func (b *CommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.CommitBa
return []sealiface.CommitBatchRes{res}, xerrors.Errorf("couldn't get miner info: %w", err)
}

maxFee := b.feeCfg.MaxCommitBatchGasFee.FeeForSectors(len(infos))

bf, err := b.api.ChainBaseFee(b.mctx, tok)
if err != nil {
return []sealiface.CommitBatchRes{res}, xerrors.Errorf("couldn't get base fee: %w", err)
Expand All @@ -304,14 +307,14 @@ func (b *CommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.CommitBa

aggFee := policy.AggregateNetworkFee(nv, len(infos), bf)

goodFunds := big.Add(b.feeCfg.MaxCommitGasFee, big.Add(collateral, aggFee))
goodFunds := big.Add(maxFee, big.Add(collateral, aggFee))

from, _, err := b.addrSel(b.mctx, mi, api.CommitAddr, goodFunds, collateral)
if err != nil {
return []sealiface.CommitBatchRes{res}, xerrors.Errorf("no good address found: %w", err)
}

mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.ProveCommitAggregate, collateral, b.feeCfg.MaxCommitGasFee, enc.Bytes())
mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.ProveCommitAggregate, collateral, maxFee, enc.Bytes())
if err != nil {
return []sealiface.CommitBatchRes{res}, xerrors.Errorf("sending message failed: %w", err)
}
Expand Down Expand Up @@ -371,14 +374,14 @@ func (b *CommitBatcher) processSingle(mi miner.MinerInfo, sn abi.SectorNumber, i
return cid.Undef, err
}

goodFunds := big.Add(collateral, b.feeCfg.MaxCommitGasFee)
goodFunds := big.Add(collateral, big.Int(b.feeCfg.MaxCommitGasFee))

from, _, err := b.addrSel(b.mctx, mi, api.CommitAddr, goodFunds, collateral)
if err != nil {
return cid.Undef, xerrors.Errorf("no good address to send commit message from: %w", err)
}

mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.ProveCommitSector, collateral, b.feeCfg.MaxCommitGasFee, enc.Bytes())
mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.ProveCommitSector, collateral, big.Int(b.feeCfg.MaxCommitGasFee), enc.Bytes())
if err != nil {
return cid.Undef, xerrors.Errorf("pushing message to mpool: %w", err)
}
Expand Down
10 changes: 6 additions & 4 deletions extern/storage-sealing/precommit_batch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
"github.com/filecoin-project/lotus/node/config"
)

type PreCommitBatcherApi interface {
Expand All @@ -40,7 +41,7 @@ type PreCommitBatcher struct {
maddr address.Address
mctx context.Context
addrSel AddrSel
feeCfg FeeConfig
feeCfg config.MinerFeeConfig
getConfig GetSealingConfigFunc

cutoffs map[abi.SectorNumber]time.Time
Expand All @@ -52,7 +53,7 @@ type PreCommitBatcher struct {
lk sync.Mutex
}

func NewPreCommitBatcher(mctx context.Context, maddr address.Address, api PreCommitBatcherApi, addrSel AddrSel, feeCfg FeeConfig, getConfig GetSealingConfigFunc) *PreCommitBatcher {
func NewPreCommitBatcher(mctx context.Context, maddr address.Address, api PreCommitBatcherApi, addrSel AddrSel, feeCfg config.MinerFeeConfig, getConfig GetSealingConfigFunc) *PreCommitBatcher {
b := &PreCommitBatcher{
api: api,
maddr: maddr,
Expand Down Expand Up @@ -227,14 +228,15 @@ func (b *PreCommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.PreCo
return []sealiface.PreCommitBatchRes{res}, xerrors.Errorf("couldn't get miner info: %w", err)
}

goodFunds := big.Add(deposit, b.feeCfg.MaxPreCommitGasFee)
maxFee := b.feeCfg.MaxPreCommitBatchGasFee.FeeForSectors(len(params.Sectors))
goodFunds := big.Add(deposit, maxFee)

from, _, err := b.addrSel(b.mctx, mi, api.PreCommitAddr, goodFunds, deposit)
if err != nil {
return []sealiface.PreCommitBatchRes{res}, xerrors.Errorf("no good address found: %w", err)
}

mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.PreCommitSectorBatch, deposit, b.feeCfg.MaxPreCommitGasFee, enc.Bytes())
mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.PreCommitSectorBatch, deposit, maxFee, enc.Bytes())
if err != nil {
return []sealiface.PreCommitBatchRes{res}, xerrors.Errorf("sending message failed: %w", err)
}
Expand Down
11 changes: 3 additions & 8 deletions extern/storage-sealing/sealing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
"github.com/filecoin-project/lotus/node/config"
)

const SectorStorePrefix = "/sectors"
Expand Down Expand Up @@ -79,7 +80,7 @@ type AddrSel func(ctx context.Context, mi miner.MinerInfo, use api.AddrUse, good

type Sealing struct {
api SealingAPI
feeCfg FeeConfig
feeCfg config.MinerFeeConfig
events Events

maddr address.Address
Expand Down Expand Up @@ -112,12 +113,6 @@ type Sealing struct {
dealInfo *CurrentDealInfoManager
}

type FeeConfig struct {
MaxPreCommitGasFee abi.TokenAmount
MaxCommitGasFee abi.TokenAmount
MaxTerminateGasFee abi.TokenAmount
}

type openSector struct {
used abi.UnpaddedPieceSize // change to bitfield/rle when AddPiece gains offset support to better fill sectors

Expand All @@ -134,7 +129,7 @@ type pendingPiece struct {
accepted func(abi.SectorNumber, abi.UnpaddedPieceSize, error)
}

func New(api SealingAPI, fc FeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing {
func New(api SealingAPI, fc config.MinerFeeConfig, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, prov ffiwrapper.Prover, pcp PreCommitPolicy, gc GetSealingConfigFunc, notifee SectorStateNotifee, as AddrSel) *Sealing {
s := &Sealing{
api: api,
feeCfg: fc,
Expand Down
8 changes: 4 additions & 4 deletions extern/storage-sealing/states_sealing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,15 +334,15 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
return nil
}

goodFunds := big.Add(deposit, m.feeCfg.MaxPreCommitGasFee)
goodFunds := big.Add(deposit, big.Int(m.feeCfg.MaxPreCommitGasFee))

from, _, err := m.addrSel(ctx.Context(), mi, api.PreCommitAddr, goodFunds, deposit)
if err != nil {
return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("no good address to send precommit message from: %w", err)})
}

log.Infof("submitting precommit for sector %d (deposit: %s): ", sector.SectorNumber, deposit)
mcid, err := m.api.SendMsg(ctx.Context(), from, m.maddr, miner.Methods.PreCommitSector, deposit, m.feeCfg.MaxPreCommitGasFee, enc.Bytes())
mcid, err := m.api.SendMsg(ctx.Context(), from, m.maddr, miner.Methods.PreCommitSector, deposit, big.Int(m.feeCfg.MaxPreCommitGasFee), enc.Bytes())
if err != nil {
if params.ReplaceCapacity {
m.remarkForUpgrade(params.ReplaceSectorNumber)
Expand Down Expand Up @@ -566,15 +566,15 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo
collateral = big.Zero()
}

goodFunds := big.Add(collateral, m.feeCfg.MaxCommitGasFee)
goodFunds := big.Add(collateral, big.Int(m.feeCfg.MaxCommitGasFee))

from, _, err := m.addrSel(ctx.Context(), mi, api.CommitAddr, goodFunds, collateral)
if err != nil {
return ctx.Send(SectorCommitFailed{xerrors.Errorf("no good address to send commit message from: %w", err)})
}

// TODO: check seed / ticket / deals are up to date
mcid, err := m.api.SendMsg(ctx.Context(), from, m.maddr, miner.Methods.ProveCommitSector, collateral, m.feeCfg.MaxCommitGasFee, enc.Bytes())
mcid, err := m.api.SendMsg(ctx.Context(), from, m.maddr, miner.Methods.ProveCommitSector, collateral, big.Int(m.feeCfg.MaxCommitGasFee), enc.Bytes())
if err != nil {
return ctx.Send(SectorCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)})
}
Expand Down
9 changes: 5 additions & 4 deletions extern/storage-sealing/terminate_batch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (

"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/node/config"
)

type TerminateBatcherApi interface {
Expand All @@ -34,7 +35,7 @@ type TerminateBatcher struct {
maddr address.Address
mctx context.Context
addrSel AddrSel
feeCfg FeeConfig
feeCfg config.MinerFeeConfig
getConfig GetSealingConfigFunc

todo map[SectorLocation]*bitfield.BitField // MinerSectorLocation -> BitField
Expand All @@ -46,7 +47,7 @@ type TerminateBatcher struct {
lk sync.Mutex
}

func NewTerminationBatcher(mctx context.Context, maddr address.Address, api TerminateBatcherApi, addrSel AddrSel, feeCfg FeeConfig, getConfig GetSealingConfigFunc) *TerminateBatcher {
func NewTerminationBatcher(mctx context.Context, maddr address.Address, api TerminateBatcherApi, addrSel AddrSel, feeCfg config.MinerFeeConfig, getConfig GetSealingConfigFunc) *TerminateBatcher {
b := &TerminateBatcher{
api: api,
maddr: maddr,
Expand Down Expand Up @@ -214,12 +215,12 @@ func (b *TerminateBatcher) processBatch(notif, after bool) (*cid.Cid, error) {
return nil, xerrors.Errorf("couldn't get miner info: %w", err)
}

from, _, err := b.addrSel(b.mctx, mi, api.TerminateSectorsAddr, b.feeCfg.MaxTerminateGasFee, b.feeCfg.MaxTerminateGasFee)
from, _, err := b.addrSel(b.mctx, mi, api.TerminateSectorsAddr, big.Int(b.feeCfg.MaxTerminateGasFee), big.Int(b.feeCfg.MaxTerminateGasFee))
if err != nil {
return nil, xerrors.Errorf("no good address found: %w", err)
}

mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.TerminateSectors, big.Zero(), b.feeCfg.MaxTerminateGasFee, enc.Bytes())
mcid, err := b.api.SendMsg(b.mctx, from, b.maddr, miner.Methods.TerminateSectors, big.Zero(), big.Int(b.feeCfg.MaxTerminateGasFee), enc.Bytes())
if err != nil {
return nil, xerrors.Errorf("sending message failed: %w", err)
}
Expand Down
34 changes: 30 additions & 4 deletions node/config/def.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (

"github.com/ipfs/go-cid"

"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"

"github.com/filecoin-project/lotus/chain/types"
Expand Down Expand Up @@ -114,9 +116,23 @@ type SealingConfig struct {
// todo TargetSectors - stop auto-pleding new sectors after this many sectors are sealed, default CC upgrade for deals sectors if above
}

type BatchFeeConfig struct {
Base types.FIL
PerSector types.FIL
}

func (b *BatchFeeConfig) FeeForSectors(nSectors int) abi.TokenAmount {
return big.Add(big.Int(b.Base), big.Mul(big.NewInt(int64(nSectors)), big.Int(b.PerSector)))
}

type MinerFeeConfig struct {
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL
MaxPreCommitGasFee types.FIL
MaxCommitGasFee types.FIL

// maxBatchFee = maxBase + maxPerSector * nSectors
MaxPreCommitBatchGasFee BatchFeeConfig
MaxCommitBatchGasFee BatchFeeConfig

MaxTerminateGasFee types.FIL
MaxWindowPoStGasFee types.FIL
MaxPublishDealsFee types.FIL
Expand Down Expand Up @@ -309,8 +325,18 @@ func DefaultStorageMiner() *StorageMiner {
},

Fees: MinerFeeConfig{
MaxPreCommitGasFee: types.MustParseFIL("0.025"),
MaxCommitGasFee: types.MustParseFIL("0.05"),
MaxPreCommitGasFee: types.MustParseFIL("0.025"),
MaxCommitGasFee: types.MustParseFIL("0.05"),

MaxPreCommitBatchGasFee: BatchFeeConfig{
Base: types.MustParseFIL("0.025"), // todo: come up with good values
PerSector: types.MustParseFIL("0.025"),
},
MaxCommitBatchGasFee: BatchFeeConfig{
Base: types.MustParseFIL("0.05"),
PerSector: types.MustParseFIL("0.05"),
},

MaxTerminateGasFee: types.MustParseFIL("0.5"),
MaxWindowPoStGasFee: types.MustParseFIL("5"),
MaxPublishDealsFee: types.MustParseFIL("0.05"),
Expand Down
Loading