all: skip noisy/faulty benchmarks + add b.ReportAllocs for every benc…

…hmark (bp #8856) (#8859) * all: skip noisy/faulty benchmarks + add b.ReportAllocs for every benchmark (#8856) * Skips very noisy benchmarks that end up running only for b.N=1 because their entire time is spent in setup, and varying parameters doesn't change much given that the number of stores is what dominates the expense. To ensure we can provide reliable benchmarks, progressively for the project, skip these until there is a proper re-work of what the benchmarks need to do * Previously sub-benchmarks: b.Run(...) did not b.ReportAllocs() due to a faulty assumption that invoking b.ReportAllocs() at the top would be inherited by all sub-benchmarks. This change fixes that Fixes #8779 Fixes #8855 Co-authored-by: Alessio Treglia <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> (cherry picked from commit b9f3db1) * Remove stray code that got pulled in by mergify but not essential Co-authored-by: Emmanuel T Odeke <[email protected]> Co-authored-by: Alessio Treglia <[email protected]>
cosmos · Mar 11, 2021 · 1d98b2a · 1d98b2a
1 parent c6462e6
commit 1d98b2a
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 0 deletions.
diff --git a/crypto/armor_test.go b/crypto/armor_test.go
@@ -162,6 +162,7 @@ func BenchmarkBcryptGenerateFromPassword(b *testing.B) {
 	for securityParam := 9; securityParam < 16; securityParam++ {
 		param := securityParam
 		b.Run(fmt.Sprintf("benchmark-security-param-%d", param), func(b *testing.B) {
+			b.ReportAllocs()
 			saltBytes := tmcrypto.CRandBytes(16)
 			b.ResetTimer()
 			for i := 0; i < b.N; i++ {

diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go
@@ -689,6 +689,9 @@ func BenchmarkMultistoreSnapshotRestore1M(b *testing.B) {
 }
 
 func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) {
+	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
+
+	b.ReportAllocs()
 	b.StopTimer()
 	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)
 	version := source.LastCommitID().Version
@@ -716,6 +719,9 @@ func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) {
 }
 
 func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys uint64) {
+	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
+
+	b.ReportAllocs()
 	b.StopTimer()
 	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)
 	version := uint64(source.LastCommitID().Version)

diff --git a/types/coin_benchmark_test.go b/types/coin_benchmark_test.go
@@ -12,6 +12,7 @@ func coinName(suffix int) string {
 func BenchmarkCoinsAdditionIntersect(b *testing.B) {
 	benchmarkingFunc := func(numCoinsA int, numCoinsB int) func(b *testing.B) {
 		return func(b *testing.B) {
+			b.ReportAllocs()
 			coinsA := Coins(make([]Coin, numCoinsA))
 			coinsB := Coins(make([]Coin, numCoinsB))
 
@@ -41,6 +42,7 @@ func BenchmarkCoinsAdditionIntersect(b *testing.B) {
 func BenchmarkCoinsAdditionNoIntersect(b *testing.B) {
 	benchmarkingFunc := func(numCoinsA int, numCoinsB int) func(b *testing.B) {
 		return func(b *testing.B) {
+			b.ReportAllocs()
 			coinsA := Coins(make([]Coin, numCoinsA))
 			coinsB := Coins(make([]Coin, numCoinsB))
 

diff --git a/x/auth/keeper/keeper_bench_test.go b/x/auth/keeper/keeper_bench_test.go
@@ -25,6 +25,7 @@ func BenchmarkAccountMapperGetAccountFound(b *testing.B) {
 }
 
 func BenchmarkAccountMapperSetAccount(b *testing.B) {
+	b.ReportAllocs()
 	app, ctx := createTestApp(false)
 
 	b.ResetTimer()

diff --git a/x/bank/bench_test.go b/x/bank/bench_test.go
@@ -60,6 +60,7 @@ func BenchmarkOneBankSendTxPerBlock(b *testing.B) {
 }
 
 func BenchmarkOneBankMultiSendTxPerBlock(b *testing.B) {
+	b.ReportAllocs()
 	// Add an account at genesis
 	acc := authtypes.BaseAccount{
 		Address: addr1.String(),