diff --git a/.circleci/config.yml b/.circleci/config.yml index 975d58b1558..e502c5fc992 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,6 +92,9 @@ jobs: - run: sudo apt-get install npm - run: command: make buildall + - run: + name: check tag and version output match + command: ./scripts/version-check.sh ./lotus - store_artifacts: path: lotus - store_artifacts: @@ -333,7 +336,7 @@ jobs: build-macos: description: build darwin lotus binary macos: - xcode: "10.0.0" + xcode: "12.5.0" working_directory: ~/go/src/github.com/filecoin-project/lotus steps: - prepare: @@ -368,6 +371,9 @@ jobs: - run: command: make build no_output_timeout: 30m + - run: + name: check tag and version output match + command: ./scripts/version-check.sh ./lotus - store_artifacts: path: lotus - store_artifacts: @@ -951,14 +957,7 @@ workflows: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - - build-macos: - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-macos - build-appimage: filters: branches: diff --git a/.circleci/template.yml b/.circleci/template.yml index 1f992e16284..131b6382cde 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -92,6 +92,9 @@ jobs: - run: sudo apt-get install npm - run: command: make buildall + - run: + name: check tag and version output match + command: ./scripts/version-check.sh ./lotus - store_artifacts: path: lotus - store_artifacts: @@ -333,7 +336,7 @@ jobs: build-macos: description: build darwin lotus binary macos: - xcode: "10.0.0" + xcode: "12.5.0" working_directory: ~/go/src/github.com/filecoin-project/lotus steps: - prepare: @@ -368,6 +371,9 @@ jobs: - run: command: make build no_output_timeout: 30m + - run: + name: check tag and version output match + command: ./scripts/version-check.sh ./lotus - store_artifacts: path: lotus - store_artifacts: @@ -806,14 +812,7 @@ workflows: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - - build-macos: - filters: - branches: - ignore: - - /.*/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ + - build-macos - build-appimage: filters: branches: diff --git a/.codecov.yml b/.codecov.yml index c266bc3ea06..01070e23086 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -40,7 +40,7 @@ coverage: - "paychmgr" miner: target: auto - threshold: 1% + threshold: 1.5% informational: false paths: - "miner" diff --git a/CHANGELOG.md b/CHANGELOG.md index 72584e87ad5..bcedad7c2c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,121 @@ # Lotus changelog +# v1.11.3 / 2021-09-29 + +lotus v1.11.3 is a feature release that's **highly recommended to ALL lotus users to upgrade**, including node +operators, storage providers and clients. It includes many improvements and bug fixes that result in perf +improvements in different area, like deal making, sealing and so on. + +## Highlights + +- 🌟🌟Introduce `MaxStagingDealsBytes - reject new deals if our staging deals area is full ([filecoin-project/lotus#7276](https://github.com/filecoin-project/lotus/pull/7276)) + - Set `MaxStagingDealsBytes` under the [Dealmaking] section of the markets' subsystem's `config.toml` to reject new incoming deals when the `deal-staging` directory of market subsystem's repo gets too large. +- 🌟🌟miner: Command to list/remove expired sectors locally ([filecoin-project/lotus#7140](https://github.com/filecoin-project/lotus/pull/7140)) + - run `./lotus-miner sectors expired -h` for more details. +- 🚀update to ffi to update-bellperson-proofs-v9-0-2 ([filecoin-project/lotus#7369](https://github.com/filecoin-project/lotus/pull/7369)) + - MinerX fellows(early testers of lotus releases) have reported faster WindowPoSt computation! +- 🌟dealpublisher: Fully validate deals before publishing ([filecoin-project/lotus#7234](https://github.com/filecoin-project/lotus/pull/7234)) + - This excludes the expired deals before sending out a PSD message which reduces the chances of PSD message failure due to invalid deals. +- 🌟Simple alert system; FD limit alerts ([filecoin-project/lotus#7108](https://github.com/filecoin-project/lotus/pull/7108)) + +## New Features + +- feat(ci): include version/cli checks in tagged releases ([filecoin-project/lotus#7331](https://github.com/filecoin-project/lotus/pull/7331)) +- Show deal sizes is sealing sectors ([filecoin-project/lotus#7261](https://github.com/filecoin-project/lotus/pull/7261)) +- config for disabling NAT port mapping ([filecoin-project/lotus#7204](https://github.com/filecoin-project/lotus/pull/7204)) +- Add optional mined block list to miner info ([filecoin-project/lotus#7202](https://github.com/filecoin-project/lotus/pull/7202)) +- Shed: Create a verifreg command for when VRK isn't a multisig ([filecoin-project/lotus#7099](https://github.com/filecoin-project/lotus/pull/7099)) + +## Improvements + +- build macOS CI ([filecoin-project/lotus#7307](https://github.com/filecoin-project/lotus/pull/7307)) +- itests: remove cid equality comparison ([filecoin-project/lotus#7292](https://github.com/filecoin-project/lotus/pull/7292)) +- Add partition info to the 'sectors status' command ([filecoin-project/lotus#7246](https://github.com/filecoin-project/lotus/pull/7246)) +- chain: Cleanup consensus logic ([filecoin-project/lotus#7255](https://github.com/filecoin-project/lotus/pull/7255)) +- builder: Handle chainstore config in ConfigFullNode ([filecoin-project/lotus#7232](https://github.com/filecoin-project/lotus/pull/7232)) +- gateway: check tipsets in ChainGetPath ([filecoin-project/lotus#7230](https://github.com/filecoin-project/lotus/pull/7230)) +- Refactor events subsystem ([filecoin-project/lotus#7000](https://github.com/filecoin-project/lotus/pull/7000)) +- test: re-enable disabled tests ([filecoin-project/lotus#7211](https://github.com/filecoin-project/lotus/pull/7211)) +- Reduce lotus-miner startup spam ([filecoin-project/lotus#7205](https://github.com/filecoin-project/lotus/pull/7205)) +- Catch deal slashed because sector was terminated ([filecoin-project/lotus#7201](https://github.com/filecoin-project/lotus/pull/7201)) +- Insert miner and network power data as gibibytes to avoid int64 overflows ([filecoin-project/lotus#7194](https://github.com/filecoin-project/lotus/pull/7194)) +- sealing: Check piece CIDs after AddPiece ([filecoin-project/lotus#7185](https://github.com/filecoin-project/lotus/pull/7185)) +- markets: OnDealExpiredOrSlashed - get deal by proposal instead of deal ID ([filecoin-project/lotus#5431](https://github.com/filecoin-project/lotus/pull/5431)) +- Incoming: improve a log message ([filecoin-project/lotus#7181](https://github.com/filecoin-project/lotus/pull/7181)) +- journal: make current log file have a fixed named (#7112) ([filecoin-project/lotus#7112](https://github.com/filecoin-project/lotus/pull/7112)) +- call string.Repeat always with positive int ([filecoin-project/lotus#7104](https://github. com/filecoin-project/lotus/pull/7104)) +- itests: support larger sector sizes; add large deal test. ([filecoin-project/lotus#7148](https://github.com/filecoin-project/lotus/pull/7148)) +- Ignore nil throttler ([filecoin-project/lotus#7169](https://github.com/filecoin-project/lotus/pull/7169)) + +## Bug Fixes + +- fix: escape periods to match actual periods in version +- fix bug for CommittedCapacitySectorLifetime ([filecoin-project/lotus#7337](https://github.com/filecoin-project/lotus/pull/7337)) +- fix a panic in HandleRecoverDealIDs ([filecoin-project/lotus#7336](https://github.com/filecoin-project/lotus/pull/7336)) +- fix index out of range ([filecoin-project/lotus#7273](https://github.com/filecoin-project/lotus/pull/7273)) +- fix: correctly handle null blocks when detecting an expensive fork ([filecoin-project/lotus#7210](https://github.com/filecoin-project/lotus/pull/7210)) +- fix: make lotus soup use the correct dependencies ([filecoin-project/lotus#7221](https://github.com/filecoin-project/lotus/pull/7221)) +- fix: init restore adds empty storage.json ([filecoin-project/lotus#7025](https://github.com/filecoin-project/lotus/pull/7025)) +- fix: disable broken testground integration test ([filecoin-project/lotus#7187](https://github.com/filecoin-project/lotus/pull/7187)) +- fix TestDealPublisher ([filecoin-project/lotus#7173](https://github.com/filecoin-project/lotus/pull/7173)) +- fix: make TestTimedCacheBlockstoreSimple pass reliably ([filecoin-project/lotus#7174](https://github.com/filecoin-project/lotus/pull/7174)) +- Fix throttling bug ([filecoin-project/lotus#7177](https://github.com/filecoin-project/lotus/pull/7177)) +- sealing: Fix sector state accounting with FinalizeEarly ([filecoin-project/lotus#7256](https://github.com/filecoin-project/lotus/pull/7256)) +- docker entrypoint.sh missing variable escape character ([filecoin-project/lotus#7291](https://github.com/filecoin-project/lotus/pull/7291)) +- sealing: Fix retry loop in SubmitCommitAggregate ([filecoin-project/lotus#7245](https://github.com/filecoin-project/lotus/pull/7245)) +- sectors expired: Handle precomitted and unproven sectors correctly ([filecoin-project/lotus#7236](https://github.com/filecoin-project/lotus/pull/7236)) +- stores: Fix reserved disk usage log spam ([filecoin-project/lotus#7233](https://github.com/filecoin-project/lotus/pull/7233)) + + +## Dependency Updates + +- github.com/filecoin-project/go-fil-markets (v1.8.1 -> v1.12.0): +- github.com/filecoin-project/go-data-transfer (v1.7.8 -> v1.10.1): +- update to ffi to update-bellperson-proofs-v9-0-2 ([filecoin-project/lotus#7369](https://github.com/filecoin-project/lotus/pull/7369)) +- fix(deps): use go-graphsync v0.9.3 with hotfix +- Update to unified go-graphsync v0.9.0 ([filecoin-project/lotus#7197](https://github.com/filecoin-project/lotus/pull/7197)) + +## Others + +- v1.11.3-rc2 ([filecoin-project/lotus#7371](https://github.com/filecoin-project/lotus/pull/7371)) +- v1.11.3-rc1 ([filecoin-project/lotus#7299](https://github.com/filecoin-project/lotus/pull/7299)) +- Increase threshold from 0.5% to 1% ([filecoin-project/lotus#7262](https://github.com/filecoin-project/lotus/pull/7262)) +- ci: exclude cruft from code coverage ([filecoin-project/lotus#7189](https://github.com/filecoin-project/lotus/pull/7189)) +- Bump version to v1.11.3-dev ([filecoin-project/lotus#7180](https://github.com/filecoin-project/lotus/pull/7180)) +- test: disable flaky TestBatchDealInput ([filecoin-project/lotus#7176](https://github.com/filecoin-project/lotus/pull/7176)) +- Turn off patch ([filecoin-project/lotus#7172](https://github.com/filecoin-project/lotus/pull/7172)) +- test: disable flaky TestSimultaneousTransferLimit ([filecoin-project/lotus#7153](https://github.com/filecoin-project/lotus/pull/7153)) + + +## Contributors + +| Contributor | Commits | Lines ± | Files Changed | +|-------------|---------|---------|---------------| +| @magik6k | 39 | +3311/-1825 | 179 | +| @Stebalien | 23 | +1935/-1417 | 84 | +| @dirkmc | 12 | +921/-732 | 111 | +| @dirkmc | 12 | +663/-790 | 30 | +| @hannahhoward | 3 | +482/-275 | 46 | +| @travisperson | 1 | +317/-65 | 5 | +| @jennijuju | 11 | +223/-126 | 24 | +| @hannahhoward | 7 | +257/-55 | 16 | +| @nonsense| 9 | +258/-37 | 19 | +| @raulk | 4 | +127/-36 | 13 | +| @raulk | 1 | +43/-60 | 15 | +| @arajasek | 4 | +74/-8 | 10 | +| @Frank | 2 | +68/-8 | 3 | +| @placer14| 2 | +52/-1 | 4 | +| @ldoublewood | 2 | +15/-13 | 3 | +| @lanzafame | 1 | +16/-2 | 1 | +| @aarshkshah1992 | 2 | +11/-6 | 2 | +| @ZenGround0 | 2 | +7/-6 | 2 | +| @ognots | 1 | +0/-10 | 2 | +| @KAYUII | 2 | +4/-4 | 2 | +| @lanzafame | 1 | +6/-0 | 1 | +| @jacobheun | 1 | +3/-3 | 1 | +| @frank | 1 | +4/-0 | 1 | + + # v1.11.2 / 2021-09-06 lotus v1.11.2 is a feature release that's **highly recommended ALL lotus users to upgrade**, including node operators, diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 957547f15ac..dc585c4860a 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 957547f15acc332a5d30b23a08f5c9dd3e58396b +Subproject commit dc585c4860a56158202161b05610d54b18b3b54a diff --git a/extern/storage-sealing/states_failed.go b/extern/storage-sealing/states_failed.go index a00fd9492ce..0c88cc38452 100644 --- a/extern/storage-sealing/states_failed.go +++ b/extern/storage-sealing/states_failed.go @@ -395,6 +395,11 @@ func (m *Sealing) HandleRecoverDealIDs(ctx Context, sector SectorInfo) error { continue } + if res.MarketDeal == nil { + failed[i] = xerrors.Errorf("nil market deal (%d,%d,%d,%s)", i, sector.SectorNumber, p.DealInfo.DealID, p.Piece.PieceCID) + continue + } + if res.MarketDeal.Proposal.PieceCID != p.Piece.PieceCID { failed[i] = xerrors.Errorf("recovered piece (%d) deal in sector %d (dealid %d) has different PieceCID %s != %s", i, sector.SectorNumber, p.DealInfo.DealID, p.Piece.PieceCID, res.MarketDeal.Proposal.PieceCID) continue diff --git a/node/config/def.go b/node/config/def.go index b75831eefef..a1a4856a907 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -101,7 +101,7 @@ func DefaultStorageMiner() *StorageMiner { PreCommitBatchWait: Duration(24 * time.Hour), // this should be less than 31.5 hours, which is the expiration of a precommit ticket PreCommitBatchSlack: Duration(3 * time.Hour), // time buffer for forceful batch submission before sectors/deals in batch would start expiring, higher value will lower the chances for message fail due to expiration - CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * policy.GetMaxSectorExpirationExtension()), + CommittedCapacitySectorLifetime: Duration(builtin.EpochDurationSeconds * uint64(policy.GetMaxSectorExpirationExtension()) * uint64(time.Second)), AggregateCommits: true, MinCommitBatch: miner5.MinAggregatedSectors, // per FIP13, we must have at least four proofs to aggregate, where 4 is the cross over point where aggregation wins out on single provecommit gas costs diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 0fd7749dadf..9e4ccc8d219 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -870,12 +870,13 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error return func(cfg sealiface.Config) (err error) { err = mutateCfg(r, func(c *config.StorageMiner) { c.Sealing = config.SealingConfig{ - MaxWaitDealsSectors: cfg.MaxWaitDealsSectors, - MaxSealingSectors: cfg.MaxSealingSectors, - MaxSealingSectorsForDeals: cfg.MaxSealingSectorsForDeals, - WaitDealsDelay: config.Duration(cfg.WaitDealsDelay), - AlwaysKeepUnsealedCopy: cfg.AlwaysKeepUnsealedCopy, - FinalizeEarly: cfg.FinalizeEarly, + MaxWaitDealsSectors: cfg.MaxWaitDealsSectors, + MaxSealingSectors: cfg.MaxSealingSectors, + MaxSealingSectorsForDeals: cfg.MaxSealingSectorsForDeals, + CommittedCapacitySectorLifetime: config.Duration(cfg.CommittedCapacitySectorLifetime), + WaitDealsDelay: config.Duration(cfg.WaitDealsDelay), + AlwaysKeepUnsealedCopy: cfg.AlwaysKeepUnsealedCopy, + FinalizeEarly: cfg.FinalizeEarly, CollateralFromMinerBalance: cfg.CollateralFromMinerBalance, AvailableBalanceBuffer: types.FIL(cfg.AvailableBalanceBuffer), @@ -904,12 +905,13 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error func ToSealingConfig(cfg *config.StorageMiner) sealiface.Config { return sealiface.Config{ - MaxWaitDealsSectors: cfg.Sealing.MaxWaitDealsSectors, - MaxSealingSectors: cfg.Sealing.MaxSealingSectors, - MaxSealingSectorsForDeals: cfg.Sealing.MaxSealingSectorsForDeals, - WaitDealsDelay: time.Duration(cfg.Sealing.WaitDealsDelay), - AlwaysKeepUnsealedCopy: cfg.Sealing.AlwaysKeepUnsealedCopy, - FinalizeEarly: cfg.Sealing.FinalizeEarly, + MaxWaitDealsSectors: cfg.Sealing.MaxWaitDealsSectors, + MaxSealingSectors: cfg.Sealing.MaxSealingSectors, + MaxSealingSectorsForDeals: cfg.Sealing.MaxSealingSectorsForDeals, + CommittedCapacitySectorLifetime: time.Duration(cfg.Sealing.CommittedCapacitySectorLifetime), + WaitDealsDelay: time.Duration(cfg.Sealing.WaitDealsDelay), + AlwaysKeepUnsealedCopy: cfg.Sealing.AlwaysKeepUnsealedCopy, + FinalizeEarly: cfg.Sealing.FinalizeEarly, CollateralFromMinerBalance: cfg.Sealing.CollateralFromMinerBalance, AvailableBalanceBuffer: types.BigInt(cfg.Sealing.AvailableBalanceBuffer), diff --git a/scripts/version-check.sh b/scripts/version-check.sh new file mode 100755 index 00000000000..4f424ca0cab --- /dev/null +++ b/scripts/version-check.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -ex + +# Validate lotus version matches the current tag +# $1 - lotus path to execute +# $2 - lotus git tag for this release +function validate_lotus_version_matches_tag(){ + # sanity checks + if [[ $# != 2 ]]; then + echo "expected 2 args for validate_lotus_version, got ${$#}" + exit 100 + fi + + # extract version from `lotus --version` response + lotus_path=$1 + # get version + lotus_raw_version=`${lotus_path} --version` + # grep for version string + lotus_actual_version=`echo ${lotus_raw_version} | grep -oE '[0-9]+\.[0-9]+\.[0-9]+'` + + # trim leading 'v' + tag=${2#v} + # trim possible -rc[0-9] + expected_version=${tag%-*} + + # check the versions are consistent + if [[ ${expected_version} != ${lotus_actual_version} ]]; then + echo "lotus version does not match build tag" + exit 101 + fi +} + +_lotus_path=$1 + +if [[ ! -z "${CIRCLE_TAG}" ]]; then + validate_lotus_version_matches_tag "${_lotus_path}" "${CIRCLE_TAG}" +else + echo "No CI tag found. Skipping version check." +fi