Add back changes to perform referential integrity tests

.github/workflows/tests.yml

@@ -3,9 +3,8 @@ on:
   workflow_call:
 
 env:
-  STACK_ORCHESTRATOR_REF: 418957a1f745c921b21286c13bb033f922a91ae9
+  STACK_ORCHESTRATOR_REF: 382aca8e42bc5e33f301f77cdd2e09cc80602fc3
   GO_ETHEREUM_REF: "v1.10.18-statediff-4.0.2-alpha"
-  IPLD_ETH_DB_REF: 91d30b9ea1acecd0a7f4307390a98bf3e289b8d7
 
 jobs:
   integrationtest:
@@ -34,19 +33,11 @@ jobs:
           ref: ${{ env.GO_ETHEREUM_REF }}
           repository: vulcanize/go-ethereum
           path: "./go-ethereum/"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ env.IPLD_ETH_DB_REF }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
       - name: Create config file
         run: |
           echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> ./config.sh
           echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-db-validator/test/contract >> ./config.sh
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
           echo db_write=$DB_WRITE >> ./config.sh
-          echo go_ethereum_db_dependency=access-node >> ./config.sh
-          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
           cat ./config.sh
       - name: Build geth
         run: |
@@ -57,10 +48,9 @@ jobs:
       - name: Run docker compose
         run: |
           docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db-sharding.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
           -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-contract.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
           --env-file "$GITHUB_WORKSPACE/config.sh" \
           up -d --build
       - name: Run integration test.
@@ -89,25 +79,12 @@ jobs:
           ref: ${{ env.STACK_ORCHESTRATOR_REF }}
           path: "./stack-orchestrator/"
           repository: vulcanize/stack-orchestrator
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ env.IPLD_ETH_DB_REF }}
-          repository: vulcanize/ipld-eth-db
-          path: "./ipld-eth-db/"
-      - name: Create config file
-        run: |
-          echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
-          echo db_write=$DB_WRITE >> ./config.sh
-          echo connecting_db_name=vulcanize_testing_v4 >> ./config.sh
-          cat ./config.sh
       - name: Run docker compose
         run: |
           docker-compose  \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-migration.yml" \
-          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-timescale-db.yml" \
-          --env-file "$GITHUB_WORKSPACE/config.sh" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db-sharding.yml" \
           up -d --build
       - name: Run unit test.
         run: |
           cd $GITHUB_WORKSPACE/ipld-eth-db-validator
-          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8066 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing_v4 make test
+          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
 	github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha
-	github.com/vulcanize/ipld-eth-server/v3 v3.2.1
 	github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha
 )
 

go.sum

@@ -1673,16 +1673,12 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
-github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.2 h1:YZ24WFrrnTC3of2EFfMIQsGUCk1hkDu++l0TIw6RrFw=
 github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.1-alpha h1:3q3siR+ZzbpJ7b0LsMsVyHj7/PJYNf+0XM48LinLUTw=
 github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.1-alpha/go.mod h1:L+0oYQrOUXO7vly1WNT9CUi1ydk5+JEW70q6EqLS4SI=
 github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha h1:gTcfBHoaRpsRbP1/dnurg5LFUaqsQMTNqs4R4S2wm9U=
 github.com/vulcanize/go-ethereum v1.10.18-statediff-4.0.2-alpha/go.mod h1:HelXH7UT1uWdb+St6UAj4pPf93GOggjIV7pVbrWIZ3o=
-github.com/vulcanize/ipfs-ethdb/v3 v3.0.3 h1:KyNMtSKZk2ZuU+mwAv/SdNEeigFkCxvsTDqxHDpwUL8=
 github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha h1:VxZONVJMxAC6vLhmlQsC3zhmY0D2A3DUoGAn8az6pXY=
 github.com/vulcanize/ipfs-ethdb/v4 v4.0.1-alpha/go.mod h1:K2ayGLsvsHcm+4A9oD3mwMLElNjw7yn70h+KXo6JdrU=
-github.com/vulcanize/ipld-eth-server/v3 v3.2.1 h1:mJSUWG0LGP8zey/i6Dm544MzESce9yZ2uHvQ5PF+hy0=
-github.com/vulcanize/ipld-eth-server/v3 v3.2.1/go.mod h1:xvnmxc6h9f70XfvEB6mV8kzTFUEdssS6gidlPWNpQeg=
 github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha h1:UDcRi9qvLWgDMKuUesEKEl/Mt94LVO0GLkgg0xVzRxs=
 github.com/vulcanize/ipld-eth-server/v4 v4.0.2-alpha/go.mod h1:k/ZzHvWBAHYmnLNY2uHIRc5WCaVPlqXiBbDMvCYlhMU=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=

pkg/validator/validator.go

@@ -138,6 +138,14 @@ func (s *service) Validate(ctx context.Context, api *ipldEth.PublicEthAPI, idxBl
 		}
 
 		s.logger.Infof("state root verified for block %d", idxBlockNum)
+
+		err = ValidateReferentialIntegrity(s.db, idxBlockNum)
+		if err != nil {
+			s.logger.Errorf("failed to verify referential integrity at block %d", idxBlockNum)
+			return idxBlockNum, err
+		}
+		s.logger.Infof("referential integrity verified for block %d", idxBlockNum)
+
 		if s.progressChan != nil {
 			s.progressChan <- idxBlockNum
 		}

scripts/run_integration_test.sh

@@ -5,10 +5,10 @@ set -o xtrace
 
 export PGPASSWORD=password
 export DATABASE_USER=vdbm
-export DATABASE_PORT=8066
+export DATABASE_PORT=8077
 export DATABASE_PASSWORD=password
 export DATABASE_HOSTNAME=127.0.0.1
-export DATABASE_NAME=vulcanize_testing_v4
+export DATABASE_NAME=vulcanize_testing
 
 # Wait for containers to be up and execute the integration test.
 while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \

test/README.md

@@ -17,7 +17,7 @@
 
     ```bash
     # In stack-orchestrator repo.
-    git checkout 418957a1f745c921b21286c13bb033f922a91ae9
+    git checkout 382aca8e42bc5e33f301f77cdd2e09cc80602fc3
     ```
 
 ## Run
@@ -55,10 +55,6 @@
       vulcanize_test_contract=~/ipld-eth-db-validator/test/contract
 
       db_write=true
-      ipld_eth_server_db_dependency=access-node
-      go_ethereum_db_dependency=access-node
-
-      connecting_db_name=vulcanize_testing_v4
       ```
 
     - Run stack-orchestrator:
@@ -69,7 +65,7 @@
 
       ./wrapper.sh \
       -e docker \
-      -d ../docker/latest/docker-compose-db.yml \
+      -d ../docker/latest/docker-compose-db-sharding.yml \
       -d ../docker/local/docker-compose-go-ethereum.yml \
       -d ../docker/local/docker-compose-contract.yml \
       -v remove \