perf: Replace runsim with Go stdlib testing (#20490)

cosmos · Jun 17, 2024 · 6335631 · 6335631
1 parent 4133dc2
commit 6335631
Show file tree

Hide file tree

Showing 23 changed files with 802 additions and 679 deletions.
diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml
@@ -15,7 +15,7 @@ jobs:
   build:
     permissions:
       contents: read # for actions/checkout to fetch code
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     if: "!contains(github.event.head_commit.message, 'skip-sims')"
     steps:
       - uses: actions/checkout@v4
@@ -24,15 +24,9 @@ jobs:
           go-version: "1.22"
           check-latest: true
       - run: make build
-      - name: Install runsim
-        run: go install github.com/cosmos/tools/cmd/[email protected]
-      - uses: actions/cache@v4
-        with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
 
   test-sim-import-export:
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     needs: [build]
     timeout-minutes: 60
     steps:
@@ -41,33 +35,26 @@ jobs:
         with:
           go-version: "1.22"
           check-latest: true
-      - uses: actions/cache@v4
-        with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
       - name: test-sim-import-export
         run: |
           make test-sim-import-export
 
   test-sim-after-import:
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     needs: [build]
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version: "1.22"
           check-latest: true
-      - uses: actions/cache@v4
-        with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
       - name: test-sim-after-import
         run: |
           make test-sim-after-import
 
-  test-sim-multi-seed-short:
-    runs-on: ubuntu-latest
+  test-sim-deterministic:
+    runs-on: large-sdk-runner
     needs: [build]
     timeout-minutes: 60
     steps:
@@ -76,18 +63,28 @@ jobs:
         with:
           go-version: "1.22"
           check-latest: true
-      - uses: actions/cache@v4
+      - name: test-sim-nondeterminism-streaming
+        run: |
+          make test-sim-nondeterminism-streaming
+
+  test-sim-multi-seed-short:
+    runs-on: large-sdk-runner
+    needs: [build]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
+          go-version: "1.22"
+          check-latest: true
       - name: test-sim-multi-seed-short
         run: |
           make test-sim-multi-seed-short
 
   sims-notify-success:
     needs:
       [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export]
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     if: ${{ success() }}
     steps:
       - name: Check out repository
@@ -115,7 +112,7 @@ jobs:
       contents: none
     needs:
       [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export]
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     if: ${{ failure() }}
     steps:
       - name: Notify Slack on failure

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -196,6 +196,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
 
 ### CLI Breaking Changes
 
+* (perf)[#20490](https://github.com/cosmos/cosmos-sdk/pull/20490) Sims: Replace runsim command with Go stdlib testing. CLI: `Commit` default true, `Lean`, `SimulateEveryOperation`, `PrintAllInvariants`, `DBBackend` params removed
 * (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `appd export` has moved with other genesis commands, use `appd genesis export` instead.
 
 ### Deprecated

diff --git a/Makefile b/Makefile
@@ -109,9 +109,6 @@ endif
 #? all: Run tools build 
 all: build
 
-# The below include contains the tools and runsim targets.
-include contrib/devtools/Makefile
-
 ###############################################################################
 ###                                  Build                                  ###
 ###############################################################################
@@ -164,7 +161,7 @@ $(MOCKS_DIR):
 	mkdir -p $(MOCKS_DIR)
 
 #? distclean: Run `make clean` and `make tools-clean`
-distclean: clean tools-clean
+distclean: clean
 
 #? clean: Clean some auto generated directory
 clean:
@@ -286,8 +283,8 @@ endif
 #? test-sim-nondeterminism: Run non-determinism test for simapp
 test-sim-nondeterminism:
 	@echo "Running non-determinism test..."
-	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
-		-NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
+		-NumBlocks=100 -BlockSize=200 -Period=0
 
 # Requires an exported plugin. See store/streaming/README.md for documentation.
 #
@@ -300,39 +297,45 @@ test-sim-nondeterminism:
 #   make test-sim-nondeterminism-streaming
 test-sim-nondeterminism-streaming:
 	@echo "Running non-determinism-streaming test..."
-	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
-		-NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h -EnableStreaming=true
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestAppStateDeterminism \
+		-NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true
 
 test-sim-custom-genesis-fast:
 	@echo "Running custom genesis simulation..."
 	@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
-	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
-		-Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -SigverifyTx=false -v -timeout 24h
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
+		-NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false
 
-test-sim-import-export: runsim
+test-sim-import-export:
 	@echo "Running application import/export simulation. This may take several minutes..."
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppImportExport
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 20m -tags='sims' -run TestAppImportExport \
+		-NumBlocks=50 -Period=5
 
-test-sim-after-import: runsim
+test-sim-after-import:
 	@echo "Running application simulation-after-import. This may take several minutes..."
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppSimulationAfterImport
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestAppSimulationAfterImport \
+		-NumBlocks=50 -Period=5
+
 
-test-sim-custom-genesis-multi-seed: runsim
+test-sim-custom-genesis-multi-seed:
 	@echo "Running multi-seed custom genesis simulation..."
 	@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Genesis=${HOME}/.simapp/config/genesis.json -SigverifyTx=false -SimAppPkg=. -ExitOnFail 400 5 TestFullAppSimulation
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
+		-NumBlocks=400 -Period=5
 
-test-sim-multi-seed-long: runsim
+test-sim-multi-seed-long:
 	@echo "Running long multi-seed application simulation. This may take awhile!"
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 500 50 TestFullAppSimulation
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=1h -tags='sims' -run TestFullAppSimulation \
+		-NumBlocks=500 -Period=50
 
-test-sim-multi-seed-short: runsim
+test-sim-multi-seed-short:
 	@echo "Running short multi-seed application simulation. This may take awhile!"
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 10 TestFullAppSimulation
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -tags='sims' -run TestFullAppSimulation \
+		-NumBlocks=50 -Period=10
 
 test-sim-benchmark-invariants:
 	@echo "Running simulation invariant benchmarks..."
-	cd ${CURRENT_DIR}/simapp && @go test -mod=readonly -benchmem -bench=BenchmarkInvariants -run=^$ \
+	cd ${CURRENT_DIR}/simapp && go test -mod=readonly -benchmem -bench=BenchmarkInvariants -tags='sims' -run=^$ \
 	-Enabled=true -NumBlocks=1000 -BlockSize=200 \
 	-Period=1 -Commit=true -Seed=57 -v -timeout 24h
 
@@ -351,6 +354,12 @@ SIM_NUM_BLOCKS ?= 500
 SIM_BLOCK_SIZE ?= 200
 SIM_COMMIT ?= true
 
+#? test-sim-fuzz: Run fuzz test for simapp
+test-sim-fuzz:
+	@echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
+#ld flags are a quick fix to make it work on current osx
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -json -tags='sims' -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
+
 #? test-sim-benchmark: Run benchmark test for simapp
 test-sim-benchmark:
 	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@@ -390,7 +399,7 @@ test-sim-profile-streaming:
 	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
 		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out -EnableStreaming=true
 
-.PHONY: test-sim-profile test-sim-benchmark
+.PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz
 
 #? benchmark: Run benchmark tests
 benchmark:

diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go
@@ -1165,3 +1165,8 @@ func (app *BaseApp) Close() error {
 
 	return errors.Join(errs...)
 }
+
+// GetBaseApp returns the pointer to itself.
+func (app *BaseApp) GetBaseApp() *BaseApp {
+	return app
+}
diff --git a/contrib/devtools/Makefile b/contrib/devtools/Makefile
diff --git a/docs/build/building-modules/14-simulator.md b/docs/build/building-modules/14-simulator.md
@@ -69,10 +69,11 @@ As you can see, the weights are predefined in this case. Options exist to overri
 Here is how one can override the above package `simappparams`.
 
 ```go reference
-https://github.com/cosmos/cosmos-sdk/blob/release/v0.50.x/Makefile#L293-L299
+https://github.com/cosmos/cosmos-sdk/blob/release/v0.51.x/Makefile#L292-L334
 ```
+The SDK simulations can be executed like normal tests in Go from the shell or within an IDE.
+Make sure that you pass the `-tags='sims` parameter to enable them and other params that make sense for your scenario.
 
-For the last test a tool called [runsim](https://github.com/cosmos/tools/tree/master/cmd/runsim) is used, this is used to parallelize go test instances, provide info to Github and slack integrations to provide information to your team on how the simulations are running.  
 
 ### Random proposal contents
 
@@ -124,3 +125,12 @@ func NewCustomApp(...) {
   ...
 }
 ```
+
+## Integration with the Go fuzzer framework
+
+The simulations provide deterministic behaviour already. The integration with the [Go fuzzer](https://go.dev/doc/security/fuzz/)
+can be done at a high level with the deterministic pseudo random number generator where the fuzzer provides varying numbers. 
+
+```go reference
+https://github.com/cosmos/cosmos-sdk/blob/release/v0.51.x/Makefile#L352-L355
+```