diff --git a/chain/actors/builtin/miner/actor.go.template b/chain/actors/builtin/miner/actor.go.template index edc298ca92a..7ffe9f146d8 100644 --- a/chain/actors/builtin/miner/actor.go.template +++ b/chain/actors/builtin/miner/actor.go.template @@ -141,10 +141,21 @@ type Deadline interface { } type Partition interface { + // AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors AllSectors() (bitfield.BitField, error) + + // Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt). + // Faults ∩ Terminated = ∅ FaultySectors() (bitfield.BitField, error) + + // Subset of faulty sectors expected to recover on next PoSt + // Recoveries ∩ Terminated = ∅ RecoveringSectors() (bitfield.BitField, error) + + // Live sectors are those that are not terminated (but may be faulty). LiveSectors() (bitfield.BitField, error) + + // Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power. ActiveSectors() (bitfield.BitField, error) } diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index 43e3a082887..4621fa48b1f 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -200,10 +200,21 @@ type Deadline interface { } type Partition interface { + // AllSectors returns all sector numbers in this partition, including faulty, unproven, and terminated sectors AllSectors() (bitfield.BitField, error) + + // Subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt). + // Faults ∩ Terminated = ∅ FaultySectors() (bitfield.BitField, error) + + // Subset of faulty sectors expected to recover on next PoSt + // Recoveries ∩ Terminated = ∅ RecoveringSectors() (bitfield.BitField, error) + + // Live sectors are those that are not terminated (but may be faulty). LiveSectors() (bitfield.BitField, error) + + // Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power. ActiveSectors() (bitfield.BitField, error) } diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index 39f9f53b98b..194b1e55416 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -44,6 +44,7 @@ var sectorsCmd = &cli.Command{ sectorsUpdateCmd, sectorsPledgeCmd, sectorsCheckExpireCmd, + sectorsExpiredCmd, sectorsRenewCmd, sectorsExtendCmd, sectorsTerminateCmd, @@ -1515,6 +1516,192 @@ var sectorsUpdateCmd = &cli.Command{ }, } +var sectorsExpiredCmd = &cli.Command{ + Name: "expired", + Usage: "Get or cleanup expired sectors", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-removed", + Usage: "show removed sectors", + }, + &cli.BoolFlag{ + Name: "remove-expired", + Usage: "remove expired sectors", + }, + + &cli.Int64Flag{ + Name: "confirm-remove-count", + Hidden: true, + }, + &cli.Int64Flag{ + Name: "expired-epoch", + Usage: "epoch at which to check sector expirations", + DefaultText: "WinningPoSt lookback epoch", + }, + }, + Action: func(cctx *cli.Context) error { + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + fullApi, nCloser, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return xerrors.Errorf("getting fullnode api: %w", err) + } + defer nCloser() + ctx := lcli.ReqContext(cctx) + + head, err := fullApi.ChainHead(ctx) + if err != nil { + return xerrors.Errorf("getting chain head: %w", err) + } + + lbEpoch := abi.ChainEpoch(cctx.Int64("expired-epoch")) + if !cctx.IsSet("expired-epoch") { + nv, err := fullApi.StateNetworkVersion(ctx, head.Key()) + if err != nil { + return xerrors.Errorf("getting network version: %w", err) + } + + lbEpoch = head.Height() - policy.GetWinningPoStSectorSetLookback(nv) + if lbEpoch < 0 { + return xerrors.Errorf("too early to terminate sectors") + } + } + + if cctx.IsSet("confirm-remove-count") && !cctx.IsSet("expired-epoch") { + return xerrors.Errorf("--expired-epoch must be specified with --confirm-remove-count") + } + + lbts, err := fullApi.ChainGetTipSetByHeight(ctx, lbEpoch, head.Key()) + if err != nil { + return xerrors.Errorf("getting lookback tipset: %w", err) + } + + maddr, err := nodeApi.ActorAddress(ctx) + if err != nil { + return xerrors.Errorf("getting actor address: %w", err) + } + + // toCheck is a working bitfield which will only contain terminated sectors + toCheck := bitfield.New() + { + sectors, err := nodeApi.SectorsList(ctx) + if err != nil { + return xerrors.Errorf("getting sector list: %w", err) + } + + for _, sector := range sectors { + toCheck.Set(uint64(sector)) + } + } + + mact, err := fullApi.StateGetActor(ctx, maddr, lbts.Key()) + if err != nil { + return err + } + + tbs := blockstore.NewTieredBstore(blockstore.NewAPIBlockstore(fullApi), blockstore.NewMemory()) + mas, err := miner.Load(adt.WrapStore(ctx, cbor.NewCborStore(tbs)), mact) + if err != nil { + return err + } + + alloc, err := mas.GetAllocatedSectors() + if err != nil { + return xerrors.Errorf("getting allocated sectors: %w", err) + } + + // only allocated sectors can be expired, + toCheck, err = bitfield.IntersectBitField(toCheck, *alloc) + if err != nil { + return xerrors.Errorf("intersecting bitfields: %w", err) + } + + if err := mas.ForEachDeadline(func(dlIdx uint64, dl miner.Deadline) error { + return dl.ForEachPartition(func(partIdx uint64, part miner.Partition) error { + live, err := part.LiveSectors() + if err != nil { + return err + } + + toCheck, err = bitfield.SubtractBitField(toCheck, live) + return err + }) + }); err != nil { + return err + } + + if cctx.Bool("remove-expired") { + color.Red("Removing sectors:\n") + } + + // toCheck now only contains sectors which either failed to precommit or are expired/terminated + fmt.Printf("Sector\tState\tExpiration\n") + + var toRemove []abi.SectorNumber + + err = toCheck.ForEach(func(u uint64) error { + s := abi.SectorNumber(u) + + st, err := nodeApi.SectorsStatus(ctx, s, true) + if err != nil { + fmt.Printf("%d:\tError getting status: %s\n", u, err) + return nil + } + + rmMsg := "" + + if st.State == api.SectorState(sealing.Removed) { + if cctx.IsSet("confirm-remove-count") || !cctx.Bool("show-removed") { + return nil + } + } else { // not removed + toRemove = append(toRemove, s) + } + + fmt.Printf("%d%s\t%s\t%s\n", s, rmMsg, st.State, lcli.EpochTime(head.Height(), st.Expiration)) + + return nil + }) + if err != nil { + return err + } + + if cctx.Bool("remove-expired") { + if !cctx.IsSet("confirm-remove-count") { + fmt.Println() + fmt.Println(color.YellowString("All"), color.GreenString("%d", len(toRemove)), color.YellowString("sectors listed above will be removed\n")) + fmt.Println(color.YellowString("To confirm removal of the above sectors, including\n all related sealed and unsealed data, run:\n")) + fmt.Println(color.RedString("lotus-miner sectors expired --remove-expired --confirm-remove-count=%d --expired-epoch=%d\n", len(toRemove), lbts.Height())) + fmt.Println(color.YellowString("WARNING: This operation is irreversible")) + return nil + } + + fmt.Println() + + if int64(len(toRemove)) != cctx.Int64("confirm-remove-count") { + return xerrors.Errorf("value of confirm-remove-count doesn't match the number of sectors which can be removed (%d)", len(toRemove)) + } + + for _, number := range toRemove { + fmt.Printf("Removing sector\t%s:\t", color.YellowString("%d", number)) + + err := nodeApi.SectorRemove(ctx, number) + if err != nil { + color.Red("ERROR: %s\n", err.Error()) + } else { + color.Green("OK\n") + } + } + } + + return nil + }, +} + var sectorsBatching = &cli.Command{ Name: "batching", Usage: "manage batch sector operations", diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 80178c8a73f..372f9deac10 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -1472,6 +1472,7 @@ COMMANDS: update-state ADVANCED: manually update the state of a sector, this may aid in error recovery pledge store random data in a sector check-expire Inspect expiring sectors + expired Get or cleanup expired sectors renew Renew expiring sectors while not exceeding each sector's max life extend Extend sector expiration terminate Terminate sector on-chain then remove (WARNING: This means losing power and collateral for the removed sector) @@ -1577,6 +1578,22 @@ OPTIONS: ``` +### lotus-miner sectors expired +``` +NAME: + lotus-miner sectors expired - Get or cleanup expired sectors + +USAGE: + lotus-miner sectors expired [command options] [arguments...] + +OPTIONS: + --show-removed show removed sectors (default: false) + --remove-expired remove expired sectors (default: false) + --expired-epoch value epoch at which to check sector expirations (default: WinningPoSt lookback epoch) + --help, -h show help (default: false) + +``` + ### lotus-miner sectors renew ``` NAME: